From 24ed9e0bf20f1b8c976c8d7bfa64aecd0fce0fe4 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:22:35 +0800 Subject: [PATCH 1/7] perf(cpp): runtime AVX2 dispatch for the ALP kernels The ALP SIMD kernels are written with SIMDe's AVX2 API, but TsFile builds for the portable x86-64 baseline, so SIMDe compiles them into its portable fallback. Mirror the RLE packer's dispatch: add native AVX2 copies of EncodeValues, DecodeValues and the bit-unpack kernels in alp_avx2.h, compiled with __attribute__((target("avx2"))), and select them at runtime through common::cpu_has_avx2(). The binary's ISA floor is unchanged; non-x86 targets, MSVC and CPUs without AVX2 keep the SIMDe path, and both paths encode identical bytes and decode bit-identical values. Measured on an i7-11700 with the default build (no -mavx2), ALP block size (1024 values) per call: float encode 3.3-6.2x, float decode 1.4-1.6x, double encode 1.2-1.6x, double decode 1.03-1.10x, uint32 unpack 2.9-3.5x, uint64 unpack 2.7-3.4x. Add differential AVX2-vs-SIMDe tests over all factor/exponent pairs, special values, counts around the vector-width boundaries and every safe bit width, plus malformed-exception status checks. Update the ALP design doc and add cpp/bench_mark/alp_avx2_benchmark.cc. --- cpp/CMakeLists.txt | 9 + cpp/bench_mark/README.md | 15 + cpp/bench_mark/alp_avx2_benchmark.cc | 415 +++++++++++++++++++++++++++ cpp/docs/alp-codec-design.md | 34 ++- cpp/src/encoding/alp/alp_avx2.h | 410 ++++++++++++++++++++++++++ cpp/src/encoding/alp/alp_simde.h | 74 ++++- cpp/test/encoding/alp_simde_test.cc | 261 +++++++++++++++++ 7 files changed, 1198 insertions(+), 20 deletions(-) create mode 100644 cpp/bench_mark/alp_avx2_benchmark.cc create mode 100644 cpp/src/encoding/alp/alp_avx2.h diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3bdf14500..7d403f493 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -486,6 +486,15 @@ if (BUILD_BENCHMARK) ${PROJECT_SRC_DIR}) target_link_libraries(alp_vs_gorilla_benchmark PRIVATE tsfile Threads::Threads) + + if (ENABLE_SIMD) + add_executable(alp_avx2_benchmark + bench_mark/alp_avx2_benchmark.cc) + target_include_directories(alp_avx2_benchmark PRIVATE + ${PROJECT_SRC_DIR}) + target_link_libraries(alp_avx2_benchmark PRIVATE + tsfile Threads::Threads) + endif () endif () if (BUILD_TOOLS) add_subdirectory(tools) diff --git a/cpp/bench_mark/README.md b/cpp/bench_mark/README.md index 14950e14b..3d3cd510e 100644 --- a/cpp/bench_mark/README.md +++ b/cpp/bench_mark/README.md @@ -54,3 +54,18 @@ meaningful results, repeat both forms and record filesystem, storage device, file sizes, compiler flags, and whether the OS page cache was warm. The checksum makes accidental short reads, metadata-load failures, or optimizer removal visible; it is not a TsFile-content checksum. + +`alp_avx2_benchmark` measures the ALP SIMD kernels: the SIMDe portable +implementation against the runtime-dispatched native AVX2 copies for +`EncodeValues`, `DecodeValues`, and bit-unpacking. It is meant to be built with +the default flags (no `-mavx2`), which is what a portable build ships: + +```bash +cmake -S cpp -B cpp/build/Release \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARK=ON -DBUILD_TEST=OFF +cmake --build cpp/build/Release --target alp_avx2_benchmark +./cpp/build/Release/alp_avx2_benchmark +``` + +On non-x86 targets and on x86 CPUs without AVX2 only the SIMDe column is +measured. The `ok` column checks that both kernels produce identical values. diff --git a/cpp/bench_mark/alp_avx2_benchmark.cc b/cpp/bench_mark/alp_avx2_benchmark.cc new file mode 100644 index 000000000..c39eb3b2e --- /dev/null +++ b/cpp/bench_mark/alp_avx2_benchmark.cc @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Micro-benchmark for the ALP SIMD kernels: SIMDe (portable fallback) versus +// the runtime-dispatched native AVX2 copies. Build with the default flags +// (no -mavx2) so the SIMDe side is what a portable build ships: +// +// cmake -DBUILD_BENCHMARK=ON -DBUILD_TEST=OFF .. +// ./alp_avx2_benchmark +// +// On non-x86 targets, or on x86 CPUs without AVX2, only the SIMDe column is +// measured. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "encoding/alp/alp_scalar.h" + +#ifdef ENABLE_SIMD +#include "encoding/alp/alp_simde.h" +#ifdef TSFILE_X86_AVX2_DISPATCH +#include "encoding/alp/alp_avx2.h" +#endif +#endif + +namespace { + +using Clock = std::chrono::steady_clock; +using storage::alp::AlpChooseFactorExponent; +using storage::alp::AlpPackBits; +using storage::alp::AlpSimdeKernel; +using storage::alp::AlpSimdeUnpackKernel; +using storage::alp::AlpTypeTraits; +#ifdef TSFILE_X86_AVX2_DISPATCH +using storage::alp::avx2_detail::AlpAvx2Kernel; +using storage::alp::avx2_detail::AlpAvx2UnpackKernel; +#endif + +const double kTargetBatchMs = 150.0; + +// Keeps the timed kernels from being optimized away when the AVX2 column is +// not compiled in: the kernels write these buffers, and main consumes the +// checksum. +volatile uint64_t g_checksum_sink = 0; + +template +void Consume(const std::vector& values) { + uint64_t checksum = 0; + for (size_t i = 0; i < values.size(); ++i) { + uint64_t bits = 0; + std::memcpy(&bits, &values[i], sizeof(T)); + checksum = checksum * 1099511628211ULL + bits; + } + g_checksum_sink ^= checksum; +} + +template +double MeasureMs(Fn fn, int repetitions) { + const Clock::time_point start = Clock::now(); + for (int i = 0; i < repetitions; ++i) { + fn(); + } + const Clock::time_point end = Clock::now(); + return std::chrono::duration_cast< + std::chrono::duration>(end - start) + .count() / + repetitions; +} + +// Runs one timing sample, calibrating the repetition count so a sample takes +// roughly kTargetBatchMs. Returns nanoseconds per value. +template +double TimeNsPerValue(Fn fn, uint32_t count) { + if (count == 0) { + return 0.0; + } + fn(); // warm up + const double single_ms = MeasureMs(fn, 1); + int repetitions = 1; + if (single_ms > 0.0) { + repetitions = static_cast(kTargetBatchMs / single_ms); + } + repetitions = std::max(1, std::min(repetitions, 100000)); + const double ms = MeasureMs(fn, repetitions); + return ms * 1.0e6 / static_cast(count); +} + +template +std::vector MakeDecimalData(uint64_t count, double scale, double offset) { + std::vector values(static_cast(count)); + for (uint64_t i = 0; i < count; ++i) { + const double raw = static_cast(i % 100000) * scale + offset; + values[static_cast(i)] = static_cast(raw); + } + return values; +} + +template +std::vector MakeRandomData(uint64_t count) { + std::vector values(static_cast(count)); + uint64_t state = 0x9e3779b97f4a7c15ULL; + for (uint64_t i = 0; i < count; ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + const uint64_t bits = state ^ (state >> 29); + if (sizeof(T) == sizeof(float)) { + uint32_t fbits = static_cast(bits); + float v = 0.0f; + std::memcpy(&v, &fbits, sizeof(v)); + if (!std::isfinite(v)) { + v = static_cast(i % 1024) * 0.001f; + } + values[static_cast(i)] = v; + } else { + double v = 0.0; + std::memcpy(&v, &bits, sizeof(v)); + if (!std::isfinite(v)) { + v = static_cast(i % 1024) * 0.001; + } + values[static_cast(i)] = v; + } + } + return values; +} + +template +std::vector MakeConstantData(uint64_t count) { + return std::vector(static_cast(count), + static_cast(3.141592653589793)); +} + +void PrintHeader(const char* title) { + std::cout << std::endl << "==== " << title << " ====" << std::endl; + std::cout << std::left << std::setw(10) << "pattern" << std::setw(8) + << "pair" << std::right << std::setw(16) << "simde(M/s)" + << std::setw(16) << "avx2(M/s)" << std::setw(12) << "speedup" + << std::setw(10) << "ok" << std::endl; +} + +void PrintRow(const std::string& pattern, const std::string& pair, + double simde_mps, double avx2_mps, bool ok) { + std::cout << std::left << std::setw(10) << pattern << std::setw(8) << pair + << std::right << std::setw(16) << std::fixed + << std::setprecision(2) << simde_mps; + if (avx2_mps > 0.0) { + std::cout << std::setw(16) << avx2_mps << std::setw(12) + << std::setprecision(2) << (avx2_mps / simde_mps); + } else { + std::cout << std::setw(16) << "-" << std::setw(12) << "-"; + } + std::cout << std::setw(10) << (ok ? "yes" : "no") << std::endl; +} + +#ifdef ENABLE_SIMD + +template +void BenchEncode(const std::string& pattern, const std::vector& values) { + typedef AlpTypeTraits Traits; + typedef typename Traits::Encoded Encoded; + const uint32_t count = static_cast(values.size()); + uint8_t factor = 0; + uint8_t exponent = 0; + if (!AlpChooseFactorExponent(&values[0], count, factor, exponent)) { + return; + } + + std::vector simde_encoded(count, 0); + std::vector avx2_encoded(count, 0); + std::vector simde_bitmap((count + 7) / 8, 0); + std::vector avx2_bitmap((count + 7) / 8, 0); + std::vector simde_exceptions(count, T()); + std::vector avx2_exceptions(count, T()); + uint32_t simde_exception_count = 0; + uint32_t avx2_exception_count = 0; +#ifndef TSFILE_X86_AVX2_DISPATCH + (void)avx2_exception_count; +#endif + const double simde_ns = TimeNsPerValue( + [&]() { + simde_exception_count = 0; + std::fill(simde_bitmap.begin(), simde_bitmap.end(), 0); + AlpSimdeKernel::EncodeValues( + &values[0], count, factor, exponent, &simde_encoded[0], + &simde_bitmap[0], &simde_exceptions[0], &simde_exception_count); + }, + count); + + double avx2_ns = 0.0; + bool ok = true; +#ifdef TSFILE_X86_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + avx2_ns = TimeNsPerValue( + [&]() { + avx2_exception_count = 0; + std::fill(avx2_bitmap.begin(), avx2_bitmap.end(), 0); + AlpAvx2Kernel::EncodeValues( + &values[0], count, factor, exponent, &avx2_encoded[0], + &avx2_bitmap[0], &avx2_exceptions[0], + &avx2_exception_count); + }, + count); + ok = simde_exception_count == avx2_exception_count && + simde_bitmap == avx2_bitmap && simde_encoded == avx2_encoded; + for (uint32_t i = 0; ok && i < simde_exception_count; ++i) { + ok = Traits::BitwiseEqual(simde_exceptions[i], avx2_exceptions[i]); + } + } +#endif + + Consume(simde_encoded); + Consume(simde_bitmap); + Consume(simde_exceptions); + Consume(avx2_encoded); + Consume(avx2_bitmap); + Consume(avx2_exceptions); + + std::string pair = std::to_string(static_cast(factor)) + "/" + + std::to_string(static_cast(exponent)); + const double simde_mps = simde_ns > 0.0 ? 1.0e3 / simde_ns : 0.0; + const double avx2_mps = avx2_ns > 0.0 ? 1.0e3 / avx2_ns : 0.0; + PrintRow(pattern, pair, simde_mps, avx2_mps, ok); +} + +template +void BenchDecode(const std::string& pattern, const std::vector& values) { + typedef AlpTypeTraits Traits; + typedef typename Traits::Encoded Encoded; + const uint32_t count = static_cast(values.size()); + uint8_t factor = 0; + uint8_t exponent = 0; + if (!AlpChooseFactorExponent(&values[0], count, factor, exponent)) { + return; + } + + std::vector encoded(count, 0); + std::vector bitmap((count + 7) / 8, 0); + std::vector exceptions(count, T()); + uint32_t exception_count = 0; + AlpSimdeKernel::EncodeValues(&values[0], count, factor, exponent, + &encoded[0], &bitmap[0], &exceptions[0], + &exception_count); + + std::vector simde_out(count, T()); + std::vector avx2_out(count, T()); + + const double simde_ns = TimeNsPerValue( + [&]() { + AlpSimdeKernel::DecodeValues( + &encoded[0], count, factor, exponent, &bitmap[0], + exception_count == 0 ? NULL : &exceptions[0], exception_count, + &simde_out[0]); + }, + count); + + double avx2_ns = 0.0; + bool ok = true; +#ifdef TSFILE_X86_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + avx2_ns = TimeNsPerValue( + [&]() { + AlpAvx2Kernel::DecodeValues( + &encoded[0], count, factor, exponent, &bitmap[0], + exception_count == 0 ? NULL : &exceptions[0], + exception_count, &avx2_out[0]); + }, + count); + for (uint32_t i = 0; ok && i < count; ++i) { + ok = Traits::BitwiseEqual(simde_out[i], avx2_out[i]); + } + } +#endif + + Consume(simde_out); + Consume(avx2_out); + + std::string pair = std::to_string(static_cast(factor)) + "/" + + std::to_string(static_cast(exponent)); + const double simde_mps = simde_ns > 0.0 ? 1.0e3 / simde_ns : 0.0; + const double avx2_mps = avx2_ns > 0.0 ? 1.0e3 / avx2_ns : 0.0; + PrintRow(pattern, pair, simde_mps, avx2_mps, ok); +} + +template +void BenchUnpack(uint8_t bit_width, uint32_t count) { + const Unsigned mask = + bit_width >= sizeof(Unsigned) * 8 + ? static_cast(~static_cast(0)) + : static_cast((static_cast(1) << bit_width) - + 1); + std::vector values(count, 0); + uint64_t state = 0x243f6a8885a308d3ULL; + for (uint32_t i = 0; i < count; ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + values[i] = static_cast(state) & mask; + } + std::vector body; + AlpPackBits(&values[0], count, bit_width, body); + + std::vector simde_out(count, 0); + std::vector avx2_out(count, 0); + + const double simde_ns = TimeNsPerValue( + [&]() { + AlpSimdeUnpackKernel::Unpack(body.data(), count, + bit_width, &simde_out[0]); + }, + count); + + double avx2_ns = 0.0; + bool ok = true; +#ifdef TSFILE_X86_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + avx2_ns = TimeNsPerValue( + [&]() { + AlpAvx2UnpackKernel::Unpack(body.data(), count, + bit_width, &avx2_out[0]); + }, + count); + ok = simde_out == avx2_out; + } +#endif + + Consume(simde_out); + Consume(avx2_out); + + std::string pair = "w=" + std::to_string(static_cast(bit_width)); + const double simde_mps = simde_ns > 0.0 ? 1.0e3 / simde_ns : 0.0; + const double avx2_mps = avx2_ns > 0.0 ? 1.0e3 / avx2_ns : 0.0; + PrintRow(sizeof(Unsigned) == 4 ? "u32" : "u64", pair, simde_mps, avx2_mps, + ok); +} + +template +void RunType(const std::string& type_name) { + const uint64_t count = storage::alp::ALP_BLOCK_SIZE; + const std::vector decimal = MakeDecimalData(count, 0.01, -1.0); + const std::vector random = MakeRandomData(count); + const std::vector constant = MakeConstantData(count); + + PrintHeader((type_name + " EncodeValues").c_str()); + BenchEncode("decimal", decimal); + BenchEncode("random", random); + BenchEncode("constant", constant); + + PrintHeader((type_name + " DecodeValues").c_str()); + BenchDecode("decimal", decimal); + BenchDecode("random", random); + BenchDecode("constant", constant); +} + +#endif // ENABLE_SIMD + +} // namespace + +int main() { +#ifndef ENABLE_SIMD + std::cout << "ENABLE_SIMD is OFF; nothing to benchmark." << std::endl; + return 0; +#else +#ifdef TSFILE_X86_AVX2_DISPATCH + if (!common::cpu_has_avx2()) { + std::cout << "CPU has no AVX2; only the SIMDe column is measured." + << std::endl; + } +#else + std::cout + << "Native AVX2 dispatch is not compiled on this target; only the " + "SIMDe column is measured." + << std::endl; +#endif + + std::cout << "ALP SIMD kernel micro-benchmark, " + << storage::alp::ALP_BLOCK_SIZE << " values per call" + << std::endl; + RunType("FLOAT"); + RunType("DOUBLE"); + + PrintHeader("Unpack (uint32_t)"); + const uint8_t u32_widths[] = {1, 2, 3, 4, 5, 8, 12, 16, 20, 24, 28, 32}; + for (size_t i = 0; i < sizeof(u32_widths); ++i) { + BenchUnpack(u32_widths[i], storage::alp::ALP_BLOCK_SIZE); + } + PrintHeader("Unpack (uint64_t)"); + const uint8_t u64_widths[] = {1, 2, 4, 8, 12, 16, 20, + 24, 32, 40, 48, 56, 57}; + for (size_t i = 0; i < sizeof(u64_widths); ++i) { + BenchUnpack(u64_widths[i], storage::alp::ALP_BLOCK_SIZE); + } + std::cout << "sink " << g_checksum_sink << std::endl; + return 0; +#endif +} diff --git a/cpp/docs/alp-codec-design.md b/cpp/docs/alp-codec-design.md index ea0f5d633..71b5337ba 100644 --- a/cpp/docs/alp-codec-design.md +++ b/cpp/docs/alp-codec-design.md @@ -152,18 +152,21 @@ be bit-exactly reproducible by the decoder. SIMDe is already a TsFile C++ dependency. The ALP kernels will use the existing `ENABLE_SIMD` switch and `TsFile::SIMDe` target. -- Write the kernels with `simde__m256i` and `simde_mm256_*`, matching the - existing encoding code style. -- The kernels are header-only and compile into baseline TsFile translation - units with `ENABLE_SIMD`; there is no separate `-mavx2` translation unit and - no runtime CPU dispatch yet. On GCC/Clang, SIMDe annotates the AVX2 helpers - with target attributes, so native AVX2 instructions may be emitted inside - outlined helper calls; on MSVC without `/arch:AVX2` SIMDe emulates them. - A dedicated AVX2 object library plus runtime dispatch is a follow-up. -- On AArch64 the same SIMDe source maps to NEON without the AVX2 flag. -- Add a scalar fallback used when SIMD is disabled or unavailable. -- Select the kernel at compile time through `ENABLE_SIMD`; there is no runtime - dispatch or function table yet. +- Write the portable kernels with `simde__m256i` and `simde_mm256_*` in + `alp_simde.h`, matching the existing encoding code style. +- The portable kernels are header-only and compile into baseline TsFile + translation units with `ENABLE_SIMD`. On AArch64 the same SIMDe source maps + to NEON without the AVX2 flag; on MSVC and on CPUs without AVX2 they are the + fallback. +- On x86 with GCC/Clang, `alp_avx2.h` carries native AVX2 copies of the + `EncodeValues`, `DecodeValues` and bit-unpack kernels, compiled with + `TSFILE_TARGET_AVX2` (`__attribute__((target("avx2")))`). The dispatchers + `AlpSimdKernel` / `AlpSimdUnpackKernel` in `alp_simde.h` select them once per + batch call through `common::cpu_has_avx2()` from `common/simd_dispatch.h`. + The library is not built with `-mavx2`, so the binary's ISA floor does not + change: a CPU without AVX2 runs the SIMDe path, and both paths must encode + to identical bytes and decode to bit-identical values. +- Add a scalar fallback used when SIMD is disabled. - The 64-bit unpack kernel reconstructs one value from a single 64-bit window, so it only handles fields up to 57 bits wide; wider fields are decoded by the scalar tail, which reads two overlapping words. Keep that limit in sync @@ -179,11 +182,14 @@ SIMDe is already a TsFile C++ dependency. The ALP kernels will use the existing - Constant blocks, single-value blocks, partial final blocks, and multi-block pages. - Scalar and SIMD must encode to identical bytes and decode to identical - values. + values. On AVX2-capable x86 machines, differential tests also compare the + dispatched AVX2 kernels against the SIMDe kernels directly. - Corrupt/truncated payload tests must return errors without out-of-bounds access. - Benchmarks compare ALP against Gorilla for decimal, normalized, smooth, - repeated, and high-entropy data. + repeated, and high-entropy data. `cpp/bench_mark/alp_avx2_benchmark.cc` + measures the SIMDe and AVX2 kernels per entry point in a default + (no `-mavx2`) build. ## References and Provenance diff --git a/cpp/src/encoding/alp/alp_avx2.h b/cpp/src/encoding/alp/alp_avx2.h new file mode 100644 index 000000000..089aa6970 --- /dev/null +++ b/cpp/src/encoding/alp/alp_avx2.h @@ -0,0 +1,410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_ALP_AVX2_H +#define ENCODING_ALP_AVX2_H + +// Native AVX2 copies of the ALP SIMD kernels. The library is built for the +// portable x86-64 baseline, so the SIMDe kernels in alp_simde.h compile into +// SIMDe's portable fallback (in particular the element-wise emulated gathers +// in the bit-unpack). These copies are compiled with +// __attribute__((target("avx2"))) and selected at runtime by +// common::cpu_has_avx2(); see common/simd_dispatch.h. The portable SIMDe +// kernels remain the fallback for non-x86 targets, MSVC, and CPUs without +// AVX2, and both paths must produce identical bytes and values. + +#include + +#include "alp_traits.h" +#include "common/simd_dispatch.h" + +#ifdef TSFILE_X86_AVX2_DISPATCH + +#include + +namespace storage { +namespace alp { +namespace avx2_detail { + +// AVX2 has no packed double -> int64 conversion (that arrives with AVX-512DQ), +// so each lane uses the scalar SSE2 conversion instruction. It truncates like +// _mm256_cvttps_epi32 and yields the "integer indefinite" value for NaN and +// out-of-range inputs; the callers range-check the result before using it. +TSFILE_TARGET_AVX2 +static inline __m256i AlpAvx2CvttpdEpi64(__m256d v) { +#if defined(__x86_64__) + const __m128d lo = _mm256_castpd256_pd128(v); + const __m128d hi = _mm256_extractf128_pd(v, 1); + const long long l0 = _mm_cvttsd_si64(lo); + const long long l1 = _mm_cvttsd_si64(_mm_unpackhi_pd(lo, lo)); + const long long h0 = _mm_cvttsd_si64(hi); + const long long h1 = _mm_cvttsd_si64(_mm_unpackhi_pd(hi, hi)); + return _mm256_setr_epi64x(l0, l1, h0, h1); +#else + // 32-bit x86 has no scalar 64-bit conversion; SIMDe's own fallback uses a + // C cast here too. + alignas(32) double lanes[4]; + _mm256_storeu_pd(lanes, v); + return _mm256_setr_epi64x( + static_cast(lanes[0]), static_cast(lanes[1]), + static_cast(lanes[2]), static_cast(lanes[3])); +#endif +} + +// Same story for int64 -> double: no packed AVX2 conversion. The scalar +// conversion is the one the scalar reference and the SIMDe fallback use, so +// the rounding is identical. +TSFILE_TARGET_AVX2 +static inline __m256d AlpAvx2Cvtepi64Pd(__m256i v) { + alignas(32) int64_t lanes[4]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(lanes), v); + return _mm256_setr_pd( + static_cast(lanes[0]), static_cast(lanes[1]), + static_cast(lanes[2]), static_cast(lanes[3])); +} + +template +struct AlpAvx2Kernel; + +template <> +struct AlpAvx2Kernel { + TSFILE_TARGET_AVX2 + static AlpStatus EncodeValues(const float* values, uint32_t count, + uint8_t factor, uint8_t exponent, + int32_t* encoded, uint8_t* bitmap, + float* exceptions, + uint32_t* exception_count) { + typedef AlpTypeTraits Traits; + const float factor_value = static_cast(ALP_FACT[factor]); + const float frac_value = ALP_FLOAT_FRAC[exponent]; + + // Keep the scalar reference's multiplication order -- (value * + // 10^exponent) * 10^-factor -- so both paths emit identical bytes. + const __m256 vexp = _mm256_set1_ps(ALP_FLOAT_EXP[exponent]); + const __m256 vfrac_factor = _mm256_set1_ps(ALP_FLOAT_FRAC[factor]); + const __m256 vmagic = _mm256_set1_ps(ALP_FLOAT_MAGIC); + const __m256 vfactor = _mm256_set1_ps(factor_value); + const __m256 vfrac = _mm256_set1_ps(frac_value); + const __m256 vlower = _mm256_set1_ps(ALP_FLOAT_ENCODING_LOWER_LIMIT); + const __m256 vupper = _mm256_set1_ps(ALP_FLOAT_ENCODING_UPPER_LIMIT); + const __m256i vnegzero = + _mm256_set1_epi32(static_cast(0x80000000u)); + const __m256i vall = _mm256_set1_epi32(-1); + + uint32_t ex_count = 0; + uint32_t i = 0; + for (; i + 7 < count; i += 8) { + const __m256 v = _mm256_loadu_ps(values + i); + const __m256 scaled = + _mm256_mul_ps(_mm256_mul_ps(v, vexp), vfrac_factor); + const __m256 rounded = + _mm256_sub_ps(_mm256_add_ps(scaled, vmagic), vmagic); + const __m256i vi = _mm256_cvttps_epi32(rounded); + const __m256 back = _mm256_mul_ps( + _mm256_mul_ps(_mm256_cvtepi32_ps(vi), vfactor), vfrac); + + const __m256 in_range = + _mm256_and_ps(_mm256_cmp_ps(scaled, vlower, _CMP_GE_OQ), + _mm256_cmp_ps(scaled, vupper, _CMP_LE_OQ)); + const __m256 rounded_in_range = + _mm256_and_ps(_mm256_cmp_ps(rounded, vlower, _CMP_GE_OQ), + _mm256_cmp_ps(rounded, vupper, _CMP_LE_OQ)); + const __m256 ordered = _mm256_cmp_ps(v, v, _CMP_ORD_Q); + const __m256 equal = _mm256_cmp_ps(back, v, _CMP_EQ_OQ); + const __m256i is_negzero = + _mm256_cmpeq_epi32(_mm256_castps_si256(v), vnegzero); + const __m256 not_negzero = + _mm256_castsi256_ps(_mm256_xor_si256(is_negzero, vall)); + const __m256 ok = _mm256_and_ps( + _mm256_and_ps(in_range, rounded_in_range), + _mm256_and_ps(ordered, _mm256_and_ps(equal, not_negzero))); + const int mask = _mm256_movemask_ps(ok); + + _mm256_storeu_si256(reinterpret_cast<__m256i*>(encoded + i), vi); + for (int lane = 0; lane < 8; ++lane) { + if ((mask & (1 << lane)) == 0) { + const uint32_t index = i + static_cast(lane); + bitmap[index >> 3] |= + static_cast(1u << (index & 7u)); + exceptions[ex_count++] = values[index]; + encoded[index] = 0; + } + } + } + + for (; i < count; ++i) { + int32_t value = 0; + if (!Traits::EncodeValue(values[i], factor, exponent, &value) || + !Traits::BitwiseEqual( + Traits::DecodeValue(value, factor, exponent), values[i])) { + bitmap[i >> 3] |= static_cast(1u << (i & 7u)); + exceptions[ex_count++] = values[i]; + } else { + encoded[i] = value; + } + } + *exception_count = ex_count; + return ALP_OK; + } + + TSFILE_TARGET_AVX2 + static AlpStatus DecodeValues(const int32_t* encoded, uint32_t count, + uint8_t factor, uint8_t exponent, + const uint8_t* bitmap, + const float* exceptions, + uint32_t exception_count, float* out) { + typedef AlpTypeTraits Traits; + const __m256 vfactor = + _mm256_set1_ps(static_cast(ALP_FACT[factor])); + const __m256 vfrac = _mm256_set1_ps(ALP_FLOAT_FRAC[exponent]); + + uint32_t i = 0; + for (; i + 7 < count; i += 8) { + const __m256i vi = _mm256_loadu_si256( + reinterpret_cast(encoded + i)); + const __m256 value = _mm256_mul_ps( + _mm256_mul_ps(_mm256_cvtepi32_ps(vi), vfactor), vfrac); + _mm256_storeu_ps(out + i, value); + } + for (; i < count; ++i) { + out[i] = Traits::DecodeValue(encoded[i], factor, exponent); + } + + uint32_t exception_index = 0; + for (uint32_t index = 0; index < count; ++index) { + if (bitmap != NULL && + (bitmap[index >> 3] & + static_cast(1u << (index & 7u))) != 0) { + if (exception_index >= exception_count) { + return ALP_MALFORMED; + } + out[index] = exceptions[exception_index++]; + } + } + return exception_index == exception_count ? ALP_OK : ALP_MALFORMED; + } +}; + +template <> +struct AlpAvx2Kernel { + TSFILE_TARGET_AVX2 + static AlpStatus EncodeValues(const double* values, uint32_t count, + uint8_t factor, uint8_t exponent, + int64_t* encoded, uint8_t* bitmap, + double* exceptions, + uint32_t* exception_count) { + typedef AlpTypeTraits Traits; + const double factor_value = static_cast(ALP_FACT[factor]); + const double frac_value = ALP_DOUBLE_FRAC[exponent]; + + // Same multiplication order as the scalar reference. + const __m256d vexp = _mm256_set1_pd(ALP_DOUBLE_EXP[exponent]); + const __m256d vfrac_factor = _mm256_set1_pd(ALP_DOUBLE_FRAC[factor]); + const __m256d vmagic = _mm256_set1_pd(ALP_DOUBLE_MAGIC); + const __m256d vfactor = _mm256_set1_pd(factor_value); + const __m256d vfrac = _mm256_set1_pd(frac_value); + const __m256d vlower = _mm256_set1_pd(ALP_DOUBLE_ENCODING_LOWER_LIMIT); + const __m256d vupper = _mm256_set1_pd(ALP_DOUBLE_ENCODING_UPPER_LIMIT); + const __m256i vnegzero = + _mm256_set1_epi64x(static_cast(0x8000000000000000ULL)); + const __m256i vall = _mm256_set1_epi32(-1); + + uint32_t ex_count = 0; + uint32_t i = 0; + for (; i + 3 < count; i += 4) { + const __m256d v = _mm256_loadu_pd(values + i); + const __m256d scaled = + _mm256_mul_pd(_mm256_mul_pd(v, vexp), vfrac_factor); + const __m256d rounded = + _mm256_sub_pd(_mm256_add_pd(scaled, vmagic), vmagic); + const __m256i vi = AlpAvx2CvttpdEpi64(rounded); + const __m256d back = _mm256_mul_pd( + _mm256_mul_pd(AlpAvx2Cvtepi64Pd(vi), vfactor), vfrac); + + const __m256d in_range = + _mm256_and_pd(_mm256_cmp_pd(scaled, vlower, _CMP_GE_OQ), + _mm256_cmp_pd(scaled, vupper, _CMP_LE_OQ)); + const __m256d rounded_in_range = + _mm256_and_pd(_mm256_cmp_pd(rounded, vlower, _CMP_GE_OQ), + _mm256_cmp_pd(rounded, vupper, _CMP_LE_OQ)); + const __m256d ordered = _mm256_cmp_pd(v, v, _CMP_ORD_Q); + const __m256d equal = _mm256_cmp_pd(back, v, _CMP_EQ_OQ); + const __m256i is_negzero = + _mm256_cmpeq_epi64(_mm256_castpd_si256(v), vnegzero); + const __m256d not_negzero = + _mm256_castsi256_pd(_mm256_xor_si256(is_negzero, vall)); + const __m256d ok = _mm256_and_pd( + _mm256_and_pd(in_range, rounded_in_range), + _mm256_and_pd(ordered, _mm256_and_pd(equal, not_negzero))); + const int mask = _mm256_movemask_pd(ok); + + _mm256_storeu_si256(reinterpret_cast<__m256i*>(encoded + i), vi); + for (int lane = 0; lane < 4; ++lane) { + if ((mask & (1 << lane)) == 0) { + const uint32_t index = i + static_cast(lane); + bitmap[index >> 3] |= + static_cast(1u << (index & 7u)); + exceptions[ex_count++] = values[index]; + encoded[index] = 0; + } + } + } + + for (; i < count; ++i) { + int64_t value = 0; + if (!Traits::EncodeValue(values[i], factor, exponent, &value) || + !Traits::BitwiseEqual( + Traits::DecodeValue(value, factor, exponent), values[i])) { + bitmap[i >> 3] |= static_cast(1u << (i & 7u)); + exceptions[ex_count++] = values[i]; + } else { + encoded[i] = value; + } + } + *exception_count = ex_count; + return ALP_OK; + } + + TSFILE_TARGET_AVX2 + static AlpStatus DecodeValues(const int64_t* encoded, uint32_t count, + uint8_t factor, uint8_t exponent, + const uint8_t* bitmap, + const double* exceptions, + uint32_t exception_count, double* out) { + typedef AlpTypeTraits Traits; + const __m256d vfactor = + _mm256_set1_pd(static_cast(ALP_FACT[factor])); + const __m256d vfrac = _mm256_set1_pd(ALP_DOUBLE_FRAC[exponent]); + + uint32_t i = 0; + for (; i + 3 < count; i += 4) { + const __m256i vi = _mm256_loadu_si256( + reinterpret_cast(encoded + i)); + const __m256d value = _mm256_mul_pd( + _mm256_mul_pd(AlpAvx2Cvtepi64Pd(vi), vfactor), vfrac); + _mm256_storeu_pd(out + i, value); + } + for (; i < count; ++i) { + out[i] = Traits::DecodeValue(encoded[i], factor, exponent); + } + + uint32_t exception_index = 0; + for (uint32_t index = 0; index < count; ++index) { + if (bitmap != NULL && + (bitmap[index >> 3] & + static_cast(1u << (index & 7u))) != 0) { + if (exception_index >= exception_count) { + return ALP_MALFORMED; + } + out[index] = exceptions[exception_index++]; + } + } + return exception_index == exception_count ? ALP_OK : ALP_MALFORMED; + } +}; + +template +struct AlpAvx2UnpackKernel; + +template <> +struct AlpAvx2UnpackKernel { + TSFILE_TARGET_AVX2 + static bool Unpack(const uint8_t* body, uint32_t count, uint8_t bit_width, + uint32_t* out) { + if (bit_width == 0) { + for (uint32_t i = 0; i < count; ++i) { + out[i] = 0; + } + return true; + } + const uint64_t mask64 = + bit_width >= 32 ? static_cast(0xFFFFFFFFu) + : ((static_cast(1) << bit_width) - 1); + const __m256i mask = _mm256_set1_epi64x(static_cast(mask64)); + const __m256i perm = _mm256_setr_epi32(0, 2, 4, 6, 0, 0, 0, 0); + for (uint32_t i = 0; i + 3 < count; i += 4) { + int32_t byte_offsets[4]; + int64_t bit_offsets[4]; + for (int lane = 0; lane < 4; ++lane) { + const uint64_t bit_pos = + static_cast(i + lane) * bit_width; + byte_offsets[lane] = static_cast(bit_pos >> 3); + bit_offsets[lane] = static_cast(bit_pos & 7u); + } + const __m128i indices = + _mm_loadu_si128(reinterpret_cast(byte_offsets)); + const __m256i windows = _mm256_i32gather_epi64( + reinterpret_cast(body), indices, 1); + const __m256i shifts = _mm256_loadu_si256( + reinterpret_cast(bit_offsets)); + const __m256i shifted = _mm256_srlv_epi64(windows, shifts); + const __m256i masked = _mm256_and_si256(shifted, mask); + const __m256i compact = _mm256_permutevar8x32_epi32(masked, perm); + _mm_storeu_si128(reinterpret_cast<__m128i*>(out + i), + _mm256_castsi256_si128(compact)); + } + return true; + } +}; + +template <> +struct AlpAvx2UnpackKernel { + TSFILE_TARGET_AVX2 + static bool Unpack(const uint8_t* body, uint32_t count, uint8_t bit_width, + uint64_t* out) { + if (bit_width == 0) { + for (uint32_t i = 0; i < count; ++i) { + out[i] = 0; + } + return true; + } + const uint64_t mask64 = + bit_width >= 64 ? ~static_cast(0) + : ((static_cast(1) << bit_width) - 1); + const __m256i mask = _mm256_set1_epi64x(static_cast(mask64)); + for (uint32_t i = 0; i + 3 < count; i += 4) { + int32_t byte_offsets[4]; + int64_t bit_offsets[4]; + for (int lane = 0; lane < 4; ++lane) { + const uint64_t bit_pos = + static_cast(i + lane) * bit_width; + byte_offsets[lane] = static_cast(bit_pos >> 3); + bit_offsets[lane] = static_cast(bit_pos & 7u); + } + const __m128i indices = + _mm_loadu_si128(reinterpret_cast(byte_offsets)); + const __m256i windows = _mm256_i32gather_epi64( + reinterpret_cast(body), indices, 1); + const __m256i shifts = _mm256_loadu_si256( + reinterpret_cast(bit_offsets)); + const __m256i shifted = _mm256_srlv_epi64(windows, shifts); + const __m256i masked = _mm256_and_si256(shifted, mask); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(out + i), masked); + } + return true; + } +}; + +} // namespace avx2_detail +} // namespace alp +} // namespace storage + +#endif // TSFILE_X86_AVX2_DISPATCH + +#endif // ENCODING_ALP_AVX2_H diff --git a/cpp/src/encoding/alp/alp_simde.h b/cpp/src/encoding/alp/alp_simde.h index a00aee6d6..7b2380e5b 100644 --- a/cpp/src/encoding/alp/alp_simde.h +++ b/cpp/src/encoding/alp/alp_simde.h @@ -23,19 +23,26 @@ #include #include "alp_traits.h" +#include "common/simd_dispatch.h" #include "simde/x86/avx2.h" #include "simde/x86/avx512/cvt.h" #include "simde/x86/avx512/cvtt.h" #include "simde/x86/sse4.1.h" +#ifdef TSFILE_X86_AVX2_DISPATCH +#include "alp_avx2.h" +#endif namespace storage { namespace alp { +// The SIMDe implementation below is the portable fallback. On x86 with +// GCC/Clang the dispatchers below pick the native AVX2 copy from alp_avx2.h at +// runtime when the CPU supports it. template -struct AlpSimdKernel; +struct AlpSimdeKernel; template <> -struct AlpSimdKernel { +struct AlpSimdeKernel { static AlpStatus EncodeValues(const float* values, uint32_t count, uint8_t factor, uint8_t exponent, int32_t* encoded, uint8_t* bitmap, @@ -162,7 +169,7 @@ struct AlpSimdKernel { }; template <> -struct AlpSimdKernel { +struct AlpSimdeKernel { static AlpStatus EncodeValues(const double* values, uint32_t count, uint8_t factor, uint8_t exponent, int64_t* encoded, uint8_t* bitmap, @@ -283,6 +290,42 @@ struct AlpSimdKernel { } }; +template +struct AlpSimdKernel { + static AlpStatus EncodeValues(const T* values, uint32_t count, + uint8_t factor, uint8_t exponent, + typename AlpTypeTraits::Encoded* encoded, + uint8_t* bitmap, T* exceptions, + uint32_t* exception_count) { +#ifdef TSFILE_X86_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + return avx2_detail::AlpAvx2Kernel::EncodeValues( + values, count, factor, exponent, encoded, bitmap, exceptions, + exception_count); + } +#endif + return AlpSimdeKernel::EncodeValues(values, count, factor, exponent, + encoded, bitmap, exceptions, + exception_count); + } + + static AlpStatus DecodeValues( + const typename AlpTypeTraits::Encoded* encoded, uint32_t count, + uint8_t factor, uint8_t exponent, const uint8_t* bitmap, + const T* exceptions, uint32_t exception_count, T* out) { +#ifdef TSFILE_X86_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + return avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, count, factor, exponent, bitmap, exceptions, + exception_count, out); + } +#endif + return AlpSimdeKernel::DecodeValues(encoded, count, factor, exponent, + bitmap, exceptions, + exception_count, out); + } +}; + template inline AlpStatus AlpSimdEncodeValues( const T* values, uint32_t count, uint8_t factor, uint8_t exponent, @@ -304,10 +347,10 @@ inline AlpStatus AlpSimdDecodeValues( } template -struct AlpSimdUnpackKernel; +struct AlpSimdeUnpackKernel; template <> -struct AlpSimdUnpackKernel { +struct AlpSimdeUnpackKernel { // A 32-bit value plus a 0..7 bit offset still fits in the gathered word. static uint8_t SafeBitWidth() { return 32; } @@ -354,7 +397,7 @@ struct AlpSimdUnpackKernel { }; template <> -struct AlpSimdUnpackKernel { +struct AlpSimdeUnpackKernel { // Each lane gathers a single 64-bit window and shifts it right by the bit // offset, so at most 64 - 7 bits survive. Wider fields are decoded by the // scalar tail, which spans two words. @@ -398,6 +441,25 @@ struct AlpSimdUnpackKernel { } }; +template +struct AlpSimdUnpackKernel { + static uint8_t SafeBitWidth() { + return AlpSimdeUnpackKernel::SafeBitWidth(); + } + + static bool Unpack(const uint8_t* body, uint32_t count, uint8_t bit_width, + Unsigned* out) { +#ifdef TSFILE_X86_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + return avx2_detail::AlpAvx2UnpackKernel::Unpack( + body, count, bit_width, out); + } +#endif + return AlpSimdeUnpackKernel::Unpack(body, count, bit_width, + out); + } +}; + template inline bool AlpSimdUnpackBits(const uint8_t* body, uint32_t count, uint8_t bit_width, Unsigned* out) { diff --git a/cpp/test/encoding/alp_simde_test.cc b/cpp/test/encoding/alp_simde_test.cc index cd67a4964..789692478 100644 --- a/cpp/test/encoding/alp_simde_test.cc +++ b/cpp/test/encoding/alp_simde_test.cc @@ -239,6 +239,267 @@ TEST(AlpSimdeTest, UnpackMatchesReferenceAcrossBitWidths) { } } +#ifdef TSFILE_X86_AVX2_DISPATCH + +// The AlpSimdeTest cases above run through the dispatched entry points, so on +// an AVX2 CPU they exercise the native kernels. These tests compare the AVX2 +// kernels against the SIMDe fallback directly, so a mismatch cannot be hidden +// by the scalar reference agreeing with both. + +template +static uint32_t EncodeWithSimde( + const std::vector& values, uint8_t factor, uint8_t exponent, + std::vector::Encoded>& encoded, + std::vector& bitmap, std::vector& exceptions) { + const uint32_t count = static_cast(values.size()); + encoded.assign(count, 0); + bitmap.assign((count + 7) / 8, 0); + exceptions.assign(count, T()); + uint32_t exception_count = 0; + EXPECT_EQ(ALP_OK, AlpSimdeKernel::EncodeValues( + values.empty() ? NULL : &values[0], count, factor, + exponent, encoded.empty() ? NULL : &encoded[0], + bitmap.empty() ? NULL : &bitmap[0], + exceptions.empty() ? NULL : &exceptions[0], + &exception_count)); + return exception_count; +} + +template +static uint32_t EncodeWithAvx2( + const std::vector& values, uint8_t factor, uint8_t exponent, + std::vector::Encoded>& encoded, + std::vector& bitmap, std::vector& exceptions) { + const uint32_t count = static_cast(values.size()); + encoded.assign(count, 0); + bitmap.assign((count + 7) / 8, 0); + exceptions.assign(count, T()); + uint32_t exception_count = 0; + EXPECT_EQ(ALP_OK, avx2_detail::AlpAvx2Kernel::EncodeValues( + values.empty() ? NULL : &values[0], count, factor, + exponent, encoded.empty() ? NULL : &encoded[0], + bitmap.empty() ? NULL : &bitmap[0], + exceptions.empty() ? NULL : &exceptions[0], + &exception_count)); + return exception_count; +} + +template +static void ExpectAvx2MatchesSimdeForPair(const std::vector& values, + uint8_t factor, uint8_t exponent) { + typedef AlpTypeTraits Traits; + typedef typename Traits::Encoded Encoded; + + std::vector simde_encoded; + std::vector simde_bitmap; + std::vector simde_exceptions; + std::vector avx2_encoded; + std::vector avx2_bitmap; + std::vector avx2_exceptions; + + const uint32_t simde_count = + EncodeWithSimde(values, factor, exponent, simde_encoded, simde_bitmap, + simde_exceptions); + const uint32_t avx2_count = EncodeWithAvx2( + values, factor, exponent, avx2_encoded, avx2_bitmap, avx2_exceptions); + ASSERT_EQ(simde_count, avx2_count); + EXPECT_EQ(simde_bitmap, avx2_bitmap); + EXPECT_EQ(simde_encoded, avx2_encoded); + for (uint32_t i = 0; i < simde_count; ++i) { + EXPECT_TRUE( + Traits::BitwiseEqual(simde_exceptions[i], avx2_exceptions[i])) + << "exception index " << i; + } + + const uint32_t count = static_cast(values.size()); + std::vector simde_decoded(count); + std::vector avx2_decoded(count); + const uint8_t* bitmap_ptr = simde_count == 0 ? NULL : &simde_bitmap[0]; + const T* exception_ptr = simde_count == 0 ? NULL : &simde_exceptions[0]; + ASSERT_EQ(ALP_OK, + AlpSimdeKernel::DecodeValues( + simde_encoded.empty() ? NULL : &simde_encoded[0], count, + factor, exponent, bitmap_ptr, exception_ptr, simde_count, + simde_decoded.empty() ? NULL : &simde_decoded[0])); + const uint32_t avx2_status = avx2_detail::AlpAvx2Kernel::DecodeValues( + simde_encoded.empty() ? NULL : &simde_encoded[0], count, factor, + exponent, bitmap_ptr, exception_ptr, simde_count, + avx2_decoded.empty() ? NULL : &avx2_decoded[0]); + ASSERT_EQ(ALP_OK, avx2_status); + for (uint32_t i = 0; i < count; ++i) { + EXPECT_TRUE(Traits::BitwiseEqual(simde_decoded[i], avx2_decoded[i])) + << "decoded index " << i; + } +} + +template +static void ExpectAvx2MatchesSimde(const std::vector& values) { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + typedef AlpTypeTraits Traits; + for (uint8_t exponent = 0; exponent <= Traits::MaxExponent(); ++exponent) { + for (uint8_t factor = 0; factor <= exponent; ++factor) { + ExpectAvx2MatchesSimdeForPair(values, factor, exponent); + } + } +} + +TEST(AlpAvx2Test, FloatValuesMatchSimde) { + std::vector values; + for (int i = 0; i < 1024; ++i) { + values.push_back(static_cast(i % 251) * 0.01f - 1.25f); + } + ExpectAvx2MatchesSimde(values); +} + +TEST(AlpAvx2Test, DoubleValuesMatchSimde) { + std::vector values; + for (int i = 0; i < 1024; ++i) { + values.push_back(static_cast(i % 509) * 0.001 - 0.25); + } + ExpectAvx2MatchesSimde(values); +} + +TEST(AlpAvx2Test, SpecialValuesMatchSimde) { + const float nan_f = std::numeric_limits::quiet_NaN(); + const double nan_d = std::numeric_limits::quiet_NaN(); + std::vector floats = {0.0f, + -0.0f, + 1.0f, + -1.0f, + nan_f, + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::denorm_min(), + std::numeric_limits::max()}; + ExpectAvx2MatchesSimde(floats); + + std::vector doubles = {0.0, + -0.0, + 1.0, + -1.0, + nan_d, + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::denorm_min(), + std::numeric_limits::max()}; + ExpectAvx2MatchesSimde(doubles); +} + +template +static void ExpectAvx2MatchesSimdeAroundVectorBoundaries() { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + const uint32_t counts[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 15, 16, 17, 23, 24, 25, 31, 32, + 33, 63, 64, 65, 127, 128, 129, 1024}; + uint64_t state = 0x9e3779b97f4a7c15ULL; + std::vector maximum(counts[sizeof(counts) / sizeof(counts[0]) - 1]); + for (size_t i = 0; i < maximum.size(); ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + const double fraction = static_cast(state % 100000) / 1000.0; + maximum[i] = static_cast(fraction - 50.0); + } + for (size_t c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) { + const std::vector values(maximum.begin(), + maximum.begin() + counts[c]); + // A pair that encodes the decimal data plus one that rejects most of + // it, so both the vector body and the exception path are covered. + ExpectAvx2MatchesSimdeForPair(values, 0, 0); + ExpectAvx2MatchesSimdeForPair(values, 3, 3); + ExpectAvx2MatchesSimdeForPair(values, 2, 2); + } +} + +TEST(AlpAvx2Test, FloatAroundVectorBoundaries) { + ExpectAvx2MatchesSimdeAroundVectorBoundaries(); +} + +TEST(AlpAvx2Test, DoubleAroundVectorBoundaries) { + ExpectAvx2MatchesSimdeAroundVectorBoundaries(); +} + +template +static void ExpectAvx2UnpackMatchesSimde(uint8_t bit_width, uint32_t count) { + const Unsigned mask = + bit_width >= sizeof(Unsigned) * 8 + ? static_cast(~static_cast(0)) + : static_cast((static_cast(1) << bit_width) - + 1); + std::vector values(count); + uint64_t state = 0x243f6a8885a308d3ULL; + for (size_t i = 0; i < values.size(); ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + values[i] = static_cast(state) & mask; + } + std::vector body; + AlpPackBits(values.empty() ? NULL : &values[0], count, bit_width, body); + const uint8_t dummy = 0; + const uint8_t* body_ptr = body.empty() ? &dummy : &body[0]; + + std::vector simde_out(count, 0); + std::vector avx2_out(count, 0); + ASSERT_TRUE(AlpSimdeUnpackKernel::Unpack( + body_ptr, count, bit_width, simde_out.data())); + ASSERT_TRUE(avx2_detail::AlpAvx2UnpackKernel::Unpack( + body_ptr, count, bit_width, avx2_out.data())); + EXPECT_EQ(simde_out, avx2_out) + << "bit_width=" << static_cast(bit_width) << " count=" << count; +} + +template +static void ExpectAvx2UnpackMatchesSimdeAcrossBitWidths() { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + const uint32_t counts[] = {0, 1, 2, 3, 4, 5, 7, 8, 9, 31, 64, 65, 1024}; + const uint8_t max_width = AlpSimdeUnpackKernel::SafeBitWidth(); + for (uint8_t bit_width = 0; bit_width <= max_width; ++bit_width) { + for (size_t c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) { + ExpectAvx2UnpackMatchesSimde(bit_width, counts[c]); + } + } +} + +TEST(AlpAvx2Test, UnpackUint32MatchesSimde) { + ExpectAvx2UnpackMatchesSimdeAcrossBitWidths(); +} + +TEST(AlpAvx2Test, UnpackUint64MatchesSimde) { + ExpectAvx2UnpackMatchesSimdeAcrossBitWidths(); +} + +TEST(AlpAvx2Test, DecodeMalformedStatusMatchesSimde) { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + // Bitmap position 1 is marked as an exception but no exception value is + // supplied: both paths must reject the block rather than read past the + // exception array. + const int32_t encoded[4] = {1, 2, 3, 4}; + const uint8_t bitmap[1] = {0x02}; + float out[4] = {0, 0, 0, 0}; + EXPECT_EQ(ALP_MALFORMED, AlpSimdeKernel::DecodeValues( + encoded, 4, 0, 0, bitmap, NULL, 0, out)); + EXPECT_EQ(ALP_MALFORMED, avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, 4, 0, 0, bitmap, NULL, 0, out)); + + // One exception value but no bitmap position to consume it. + const float exceptions[1] = {5.0f}; + EXPECT_EQ(ALP_MALFORMED, AlpSimdeKernel::DecodeValues( + encoded, 4, 0, 0, bitmap, exceptions, 2, out)); + EXPECT_EQ(ALP_MALFORMED, avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, 4, 0, 0, bitmap, exceptions, 2, out)); +} + +#endif // TSFILE_X86_AVX2_DISPATCH + #endif // ENABLE_SIMD } // namespace alp From 95a4a6a308b98bdb08140b95e0478e46be944403 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:55:16 +0800 Subject: [PATCH 2/7] test(cpp): apply ALP AVX2 review feedback Document the padded-body contract the gather-based unpack kernels rely on, compare the AVX2 unpack output against the packed input instead of only against SIMDe, and use nullptr in the new code. --- cpp/src/encoding/alp/alp_avx2.h | 10 +++++-- cpp/test/encoding/alp_simde_test.cc | 43 +++++++++++++++++------------ 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/cpp/src/encoding/alp/alp_avx2.h b/cpp/src/encoding/alp/alp_avx2.h index 089aa6970..6cb9a36b2 100644 --- a/cpp/src/encoding/alp/alp_avx2.h +++ b/cpp/src/encoding/alp/alp_avx2.h @@ -188,7 +188,7 @@ struct AlpAvx2Kernel { uint32_t exception_index = 0; for (uint32_t index = 0; index < count; ++index) { - if (bitmap != NULL && + if (bitmap != nullptr && (bitmap[index >> 3] & static_cast(1u << (index & 7u))) != 0) { if (exception_index >= exception_count) { @@ -306,7 +306,7 @@ struct AlpAvx2Kernel { uint32_t exception_index = 0; for (uint32_t index = 0; index < count; ++index) { - if (bitmap != NULL && + if (bitmap != nullptr && (bitmap[index >> 3] & static_cast(1u << (index & 7u))) != 0) { if (exception_index >= exception_count) { @@ -319,6 +319,12 @@ struct AlpAvx2Kernel { } }; +// Both unpack kernels gather one 64-bit window per lane, so a lane can read up +// to 8 bytes starting at that value's byte offset. Callers must pass the padded +// ALP body produced by AlpPackBits, which reserves at least 16 zero guard bytes +// past the logical stream; AlpDecodeBlock validates body_bytes against that +// padded size before unpacking. The kernels process whole groups of four only: +// AlpUnpackBits passes count & ~3 and decodes the remaining values itself. template struct AlpAvx2UnpackKernel; diff --git a/cpp/test/encoding/alp_simde_test.cc b/cpp/test/encoding/alp_simde_test.cc index 789692478..b842ad7d2 100644 --- a/cpp/test/encoding/alp_simde_test.cc +++ b/cpp/test/encoding/alp_simde_test.cc @@ -257,10 +257,10 @@ static uint32_t EncodeWithSimde( exceptions.assign(count, T()); uint32_t exception_count = 0; EXPECT_EQ(ALP_OK, AlpSimdeKernel::EncodeValues( - values.empty() ? NULL : &values[0], count, factor, - exponent, encoded.empty() ? NULL : &encoded[0], - bitmap.empty() ? NULL : &bitmap[0], - exceptions.empty() ? NULL : &exceptions[0], + values.empty() ? nullptr : &values[0], count, factor, + exponent, encoded.empty() ? nullptr : &encoded[0], + bitmap.empty() ? nullptr : &bitmap[0], + exceptions.empty() ? nullptr : &exceptions[0], &exception_count)); return exception_count; } @@ -276,10 +276,10 @@ static uint32_t EncodeWithAvx2( exceptions.assign(count, T()); uint32_t exception_count = 0; EXPECT_EQ(ALP_OK, avx2_detail::AlpAvx2Kernel::EncodeValues( - values.empty() ? NULL : &values[0], count, factor, - exponent, encoded.empty() ? NULL : &encoded[0], - bitmap.empty() ? NULL : &bitmap[0], - exceptions.empty() ? NULL : &exceptions[0], + values.empty() ? nullptr : &values[0], count, factor, + exponent, encoded.empty() ? nullptr : &encoded[0], + bitmap.empty() ? nullptr : &bitmap[0], + exceptions.empty() ? nullptr : &exceptions[0], &exception_count)); return exception_count; } @@ -314,17 +314,17 @@ static void ExpectAvx2MatchesSimdeForPair(const std::vector& values, const uint32_t count = static_cast(values.size()); std::vector simde_decoded(count); std::vector avx2_decoded(count); - const uint8_t* bitmap_ptr = simde_count == 0 ? NULL : &simde_bitmap[0]; - const T* exception_ptr = simde_count == 0 ? NULL : &simde_exceptions[0]; + const uint8_t* bitmap_ptr = simde_count == 0 ? nullptr : &simde_bitmap[0]; + const T* exception_ptr = simde_count == 0 ? nullptr : &simde_exceptions[0]; ASSERT_EQ(ALP_OK, AlpSimdeKernel::DecodeValues( - simde_encoded.empty() ? NULL : &simde_encoded[0], count, + simde_encoded.empty() ? nullptr : &simde_encoded[0], count, factor, exponent, bitmap_ptr, exception_ptr, simde_count, - simde_decoded.empty() ? NULL : &simde_decoded[0])); + simde_decoded.empty() ? nullptr : &simde_decoded[0])); const uint32_t avx2_status = avx2_detail::AlpAvx2Kernel::DecodeValues( - simde_encoded.empty() ? NULL : &simde_encoded[0], count, factor, + simde_encoded.empty() ? nullptr : &simde_encoded[0], count, factor, exponent, bitmap_ptr, exception_ptr, simde_count, - avx2_decoded.empty() ? NULL : &avx2_decoded[0]); + avx2_decoded.empty() ? nullptr : &avx2_decoded[0]); ASSERT_EQ(ALP_OK, avx2_status); for (uint32_t i = 0; i < count; ++i) { EXPECT_TRUE(Traits::BitwiseEqual(simde_decoded[i], avx2_decoded[i])) @@ -437,7 +437,7 @@ static void ExpectAvx2UnpackMatchesSimde(uint8_t bit_width, uint32_t count) { values[i] = static_cast(state) & mask; } std::vector body; - AlpPackBits(values.empty() ? NULL : &values[0], count, bit_width, body); + AlpPackBits(values.empty() ? nullptr : &values[0], count, bit_width, body); const uint8_t dummy = 0; const uint8_t* body_ptr = body.empty() ? &dummy : &body[0]; @@ -449,6 +449,15 @@ static void ExpectAvx2UnpackMatchesSimde(uint8_t bit_width, uint32_t count) { body_ptr, count, bit_width, avx2_out.data())); EXPECT_EQ(simde_out, avx2_out) << "bit_width=" << static_cast(bit_width) << " count=" << count; + // The kernels only cover whole groups of four; compare every value they + // did process against the input that was packed, so a mistake shared by + // both implementations cannot hide behind their agreement. + const uint32_t processed = count & ~static_cast(3); + for (uint32_t i = 0; i < processed; ++i) { + EXPECT_EQ(values[i], avx2_out[i]) + << "bit_width=" << static_cast(bit_width) << " count=" << count + << " index=" << i; + } } template @@ -486,9 +495,9 @@ TEST(AlpAvx2Test, DecodeMalformedStatusMatchesSimde) { const uint8_t bitmap[1] = {0x02}; float out[4] = {0, 0, 0, 0}; EXPECT_EQ(ALP_MALFORMED, AlpSimdeKernel::DecodeValues( - encoded, 4, 0, 0, bitmap, NULL, 0, out)); + encoded, 4, 0, 0, bitmap, nullptr, 0, out)); EXPECT_EQ(ALP_MALFORMED, avx2_detail::AlpAvx2Kernel::DecodeValues( - encoded, 4, 0, 0, bitmap, NULL, 0, out)); + encoded, 4, 0, 0, bitmap, nullptr, 0, out)); // One exception value but no bitmap position to consume it. const float exceptions[1] = {5.0f}; From 3842b0ced0c1d67def33d98a118ffe27adc3a3e9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:26:14 +0800 Subject: [PATCH 3/7] fix(cpp): address ALP AVX2 review feedback (round 2) Restrict ALP AVX2 dispatch to x86-64 so 32-bit x86 cannot reach an undefined double-to-int64 cast, always pass the exception bitmap to the differential decode test so a wrongly set bit with exception_count == 0 is caught, use nullptr in the benchmark, and use an epi64 all-ones mask in the double kernel. --- cpp/bench_mark/alp_avx2_benchmark.cc | 18 +++++++++--------- cpp/docs/alp-codec-design.md | 2 +- cpp/src/encoding/alp/alp_avx2.h | 18 +++++++----------- cpp/src/encoding/alp/alp_simde.h | 6 +++--- cpp/test/encoding/alp_simde_test.cc | 7 ++++--- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/cpp/bench_mark/alp_avx2_benchmark.cc b/cpp/bench_mark/alp_avx2_benchmark.cc index c39eb3b2e..b6d73eeea 100644 --- a/cpp/bench_mark/alp_avx2_benchmark.cc +++ b/cpp/bench_mark/alp_avx2_benchmark.cc @@ -55,7 +55,7 @@ using storage::alp::AlpPackBits; using storage::alp::AlpSimdeKernel; using storage::alp::AlpSimdeUnpackKernel; using storage::alp::AlpTypeTraits; -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH using storage::alp::avx2_detail::AlpAvx2Kernel; using storage::alp::avx2_detail::AlpAvx2UnpackKernel; #endif @@ -195,7 +195,7 @@ void BenchEncode(const std::string& pattern, const std::vector& values) { std::vector avx2_exceptions(count, T()); uint32_t simde_exception_count = 0; uint32_t avx2_exception_count = 0; -#ifndef TSFILE_X86_AVX2_DISPATCH +#ifndef ALP_AVX2_DISPATCH (void)avx2_exception_count; #endif const double simde_ns = TimeNsPerValue( @@ -210,7 +210,7 @@ void BenchEncode(const std::string& pattern, const std::vector& values) { double avx2_ns = 0.0; bool ok = true; -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH if (common::cpu_has_avx2()) { avx2_ns = TimeNsPerValue( [&]() { @@ -270,20 +270,20 @@ void BenchDecode(const std::string& pattern, const std::vector& values) { [&]() { AlpSimdeKernel::DecodeValues( &encoded[0], count, factor, exponent, &bitmap[0], - exception_count == 0 ? NULL : &exceptions[0], exception_count, - &simde_out[0]); + exception_count == 0 ? nullptr : &exceptions[0], + exception_count, &simde_out[0]); }, count); double avx2_ns = 0.0; bool ok = true; -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH if (common::cpu_has_avx2()) { avx2_ns = TimeNsPerValue( [&]() { AlpAvx2Kernel::DecodeValues( &encoded[0], count, factor, exponent, &bitmap[0], - exception_count == 0 ? NULL : &exceptions[0], + exception_count == 0 ? nullptr : &exceptions[0], exception_count, &avx2_out[0]); }, count); @@ -331,7 +331,7 @@ void BenchUnpack(uint8_t bit_width, uint32_t count) { double avx2_ns = 0.0; bool ok = true; -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH if (common::cpu_has_avx2()) { avx2_ns = TimeNsPerValue( [&]() { @@ -380,7 +380,7 @@ int main() { std::cout << "ENABLE_SIMD is OFF; nothing to benchmark." << std::endl; return 0; #else -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH if (!common::cpu_has_avx2()) { std::cout << "CPU has no AVX2; only the SIMDe column is measured." << std::endl; diff --git a/cpp/docs/alp-codec-design.md b/cpp/docs/alp-codec-design.md index 71b5337ba..98c691bd3 100644 --- a/cpp/docs/alp-codec-design.md +++ b/cpp/docs/alp-codec-design.md @@ -158,7 +158,7 @@ SIMDe is already a TsFile C++ dependency. The ALP kernels will use the existing translation units with `ENABLE_SIMD`. On AArch64 the same SIMDe source maps to NEON without the AVX2 flag; on MSVC and on CPUs without AVX2 they are the fallback. -- On x86 with GCC/Clang, `alp_avx2.h` carries native AVX2 copies of the +- On x86-64 with GCC/Clang, `alp_avx2.h` carries native AVX2 copies of the `EncodeValues`, `DecodeValues` and bit-unpack kernels, compiled with `TSFILE_TARGET_AVX2` (`__attribute__((target("avx2")))`). The dispatchers `AlpSimdKernel` / `AlpSimdUnpackKernel` in `alp_simde.h` select them once per diff --git a/cpp/src/encoding/alp/alp_avx2.h b/cpp/src/encoding/alp/alp_avx2.h index 6cb9a36b2..3b2d05693 100644 --- a/cpp/src/encoding/alp/alp_avx2.h +++ b/cpp/src/encoding/alp/alp_avx2.h @@ -35,6 +35,11 @@ #include "common/simd_dispatch.h" #ifdef TSFILE_X86_AVX2_DISPATCH +// These kernels use the 64-bit scalar conversion instruction, which 32-bit x86 +// does not provide without undefined behavior, so ALP dispatch is x86-64 only. +// The RLE packers keep the wider TSFILE_X86_AVX2_DISPATCH definition. +#if defined(__x86_64__) +#define ALP_AVX2_DISPATCH 1 #include @@ -48,7 +53,6 @@ namespace avx2_detail { // out-of-range inputs; the callers range-check the result before using it. TSFILE_TARGET_AVX2 static inline __m256i AlpAvx2CvttpdEpi64(__m256d v) { -#if defined(__x86_64__) const __m128d lo = _mm256_castpd256_pd128(v); const __m128d hi = _mm256_extractf128_pd(v, 1); const long long l0 = _mm_cvttsd_si64(lo); @@ -56,15 +60,6 @@ static inline __m256i AlpAvx2CvttpdEpi64(__m256d v) { const long long h0 = _mm_cvttsd_si64(hi); const long long h1 = _mm_cvttsd_si64(_mm_unpackhi_pd(hi, hi)); return _mm256_setr_epi64x(l0, l1, h0, h1); -#else - // 32-bit x86 has no scalar 64-bit conversion; SIMDe's own fallback uses a - // C cast here too. - alignas(32) double lanes[4]; - _mm256_storeu_pd(lanes, v); - return _mm256_setr_epi64x( - static_cast(lanes[0]), static_cast(lanes[1]), - static_cast(lanes[2]), static_cast(lanes[3])); -#endif } // Same story for int64 -> double: no packed AVX2 conversion. The scalar @@ -223,7 +218,7 @@ struct AlpAvx2Kernel { const __m256d vupper = _mm256_set1_pd(ALP_DOUBLE_ENCODING_UPPER_LIMIT); const __m256i vnegzero = _mm256_set1_epi64x(static_cast(0x8000000000000000ULL)); - const __m256i vall = _mm256_set1_epi32(-1); + const __m256i vall = _mm256_set1_epi64x(-1); uint32_t ex_count = 0; uint32_t i = 0; @@ -411,6 +406,7 @@ struct AlpAvx2UnpackKernel { } // namespace alp } // namespace storage +#endif // __x86_64__ #endif // TSFILE_X86_AVX2_DISPATCH #endif // ENCODING_ALP_AVX2_H diff --git a/cpp/src/encoding/alp/alp_simde.h b/cpp/src/encoding/alp/alp_simde.h index 7b2380e5b..22b085cb0 100644 --- a/cpp/src/encoding/alp/alp_simde.h +++ b/cpp/src/encoding/alp/alp_simde.h @@ -297,7 +297,7 @@ struct AlpSimdKernel { typename AlpTypeTraits::Encoded* encoded, uint8_t* bitmap, T* exceptions, uint32_t* exception_count) { -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH if (common::cpu_has_avx2()) { return avx2_detail::AlpAvx2Kernel::EncodeValues( values, count, factor, exponent, encoded, bitmap, exceptions, @@ -313,7 +313,7 @@ struct AlpSimdKernel { const typename AlpTypeTraits::Encoded* encoded, uint32_t count, uint8_t factor, uint8_t exponent, const uint8_t* bitmap, const T* exceptions, uint32_t exception_count, T* out) { -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH if (common::cpu_has_avx2()) { return avx2_detail::AlpAvx2Kernel::DecodeValues( encoded, count, factor, exponent, bitmap, exceptions, @@ -449,7 +449,7 @@ struct AlpSimdUnpackKernel { static bool Unpack(const uint8_t* body, uint32_t count, uint8_t bit_width, Unsigned* out) { -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH if (common::cpu_has_avx2()) { return avx2_detail::AlpAvx2UnpackKernel::Unpack( body, count, bit_width, out); diff --git a/cpp/test/encoding/alp_simde_test.cc b/cpp/test/encoding/alp_simde_test.cc index b842ad7d2..515973fb8 100644 --- a/cpp/test/encoding/alp_simde_test.cc +++ b/cpp/test/encoding/alp_simde_test.cc @@ -239,7 +239,7 @@ TEST(AlpSimdeTest, UnpackMatchesReferenceAcrossBitWidths) { } } -#ifdef TSFILE_X86_AVX2_DISPATCH +#ifdef ALP_AVX2_DISPATCH // The AlpSimdeTest cases above run through the dispatched entry points, so on // an AVX2 CPU they exercise the native kernels. These tests compare the AVX2 @@ -314,7 +314,8 @@ static void ExpectAvx2MatchesSimdeForPair(const std::vector& values, const uint32_t count = static_cast(values.size()); std::vector simde_decoded(count); std::vector avx2_decoded(count); - const uint8_t* bitmap_ptr = simde_count == 0 ? nullptr : &simde_bitmap[0]; + const uint8_t* bitmap_ptr = + simde_bitmap.empty() ? nullptr : &simde_bitmap[0]; const T* exception_ptr = simde_count == 0 ? nullptr : &simde_exceptions[0]; ASSERT_EQ(ALP_OK, AlpSimdeKernel::DecodeValues( @@ -507,7 +508,7 @@ TEST(AlpAvx2Test, DecodeMalformedStatusMatchesSimde) { encoded, 4, 0, 0, bitmap, exceptions, 2, out)); } -#endif // TSFILE_X86_AVX2_DISPATCH +#endif // ALP_AVX2_DISPATCH #endif // ENABLE_SIMD From a3545fe08e89ebcfd18ee41d8314c51b57380fee Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:27:54 +0800 Subject: [PATCH 4/7] docs(cpp): align ALP AVX2 wording with x86-64-only dispatch The second review round restricted the ALP AVX2 kernels to x86-64; update the remaining comments, benchmark output and docs that still referred to x86 generally. --- cpp/bench_mark/README.md | 4 ++-- cpp/bench_mark/alp_avx2_benchmark.cc | 11 +++++------ cpp/docs/alp-codec-design.md | 12 +++++++----- cpp/src/encoding/alp/alp_avx2.h | 2 +- cpp/src/encoding/alp/alp_simde.h | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cpp/bench_mark/README.md b/cpp/bench_mark/README.md index e39e59e18..fd0f54e22 100644 --- a/cpp/bench_mark/README.md +++ b/cpp/bench_mark/README.md @@ -67,8 +67,8 @@ cmake --build cpp/build/Release --target alp_avx2_benchmark ./cpp/build/Release/alp_avx2_benchmark ``` -On non-x86 targets and on x86 CPUs without AVX2 only the SIMDe column is -measured. The `ok` column checks that both kernels produce identical values. +On non-x86-64 targets, and on x86-64 CPUs without AVX2, only the SIMDe column +is measured. The `ok` column checks that both kernels produce identical values. `gorilla_v1_batch_decode_benchmark` compares the per-value GORILLA_V1 decode loop with the raw-pointer batch decoder on the same encoded stream for FLOAT diff --git a/cpp/bench_mark/alp_avx2_benchmark.cc b/cpp/bench_mark/alp_avx2_benchmark.cc index b6d73eeea..fa1a20c86 100644 --- a/cpp/bench_mark/alp_avx2_benchmark.cc +++ b/cpp/bench_mark/alp_avx2_benchmark.cc @@ -24,8 +24,8 @@ // cmake -DBUILD_BENCHMARK=ON -DBUILD_TEST=OFF .. // ./alp_avx2_benchmark // -// On non-x86 targets, or on x86 CPUs without AVX2, only the SIMDe column is -// measured. +// When ALP AVX2 dispatch is not compiled (non-x86-64 targets) or the CPU has no +// AVX2, only the SIMDe column is measured. #include #include @@ -386,10 +386,9 @@ int main() { << std::endl; } #else - std::cout - << "Native AVX2 dispatch is not compiled on this target; only the " - "SIMDe column is measured." - << std::endl; + std::cout << "ALP AVX2 dispatch is not compiled on this target; only the " + "SIMDe column is measured." + << std::endl; #endif std::cout << "ALP SIMD kernel micro-benchmark, " diff --git a/cpp/docs/alp-codec-design.md b/cpp/docs/alp-codec-design.md index 98c691bd3..2f7503ed1 100644 --- a/cpp/docs/alp-codec-design.md +++ b/cpp/docs/alp-codec-design.md @@ -156,13 +156,15 @@ SIMDe is already a TsFile C++ dependency. The ALP kernels will use the existing `alp_simde.h`, matching the existing encoding code style. - The portable kernels are header-only and compile into baseline TsFile translation units with `ENABLE_SIMD`. On AArch64 the same SIMDe source maps - to NEON without the AVX2 flag; on MSVC and on CPUs without AVX2 they are the - fallback. + to NEON without the AVX2 flag; on MSVC, 32-bit x86 and CPUs without AVX2 + they are the fallback. - On x86-64 with GCC/Clang, `alp_avx2.h` carries native AVX2 copies of the `EncodeValues`, `DecodeValues` and bit-unpack kernels, compiled with - `TSFILE_TARGET_AVX2` (`__attribute__((target("avx2")))`). The dispatchers - `AlpSimdKernel` / `AlpSimdUnpackKernel` in `alp_simde.h` select them once per - batch call through `common::cpu_has_avx2()` from `common/simd_dispatch.h`. + `TSFILE_TARGET_AVX2` (`__attribute__((target("avx2")))`). ALP AVX2 is + compiled only for x86-64 (`ALP_AVX2_DISPATCH`); 32-bit x86 keeps the SIMDe + path. The dispatchers `AlpSimdKernel` / `AlpSimdUnpackKernel` in + `alp_simde.h` select the AVX2 copies once per batch call through + `common::cpu_has_avx2()` from `common/simd_dispatch.h`. The library is not built with `-mavx2`, so the binary's ISA floor does not change: a CPU without AVX2 runs the SIMDe path, and both paths must encode to identical bytes and decode to bit-identical values. diff --git a/cpp/src/encoding/alp/alp_avx2.h b/cpp/src/encoding/alp/alp_avx2.h index 3b2d05693..4f5044bb7 100644 --- a/cpp/src/encoding/alp/alp_avx2.h +++ b/cpp/src/encoding/alp/alp_avx2.h @@ -26,7 +26,7 @@ // in the bit-unpack). These copies are compiled with // __attribute__((target("avx2"))) and selected at runtime by // common::cpu_has_avx2(); see common/simd_dispatch.h. The portable SIMDe -// kernels remain the fallback for non-x86 targets, MSVC, and CPUs without +// kernels remain the fallback for non-x86-64 targets, MSVC, and CPUs without // AVX2, and both paths must produce identical bytes and values. #include diff --git a/cpp/src/encoding/alp/alp_simde.h b/cpp/src/encoding/alp/alp_simde.h index 22b085cb0..4e4bbd1e4 100644 --- a/cpp/src/encoding/alp/alp_simde.h +++ b/cpp/src/encoding/alp/alp_simde.h @@ -35,7 +35,7 @@ namespace storage { namespace alp { -// The SIMDe implementation below is the portable fallback. On x86 with +// The SIMDe implementation below is the portable fallback. On x86-64 with // GCC/Clang the dispatchers below pick the native AVX2 copy from alp_avx2.h at // runtime when the CPU supports it. template From e5069c7b9dcd2c890f49815a3885521a34719513 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:39:29 +0800 Subject: [PATCH 5/7] test(cpp): pin ALP AVX2 int64-to-double conversion extremes The AVX2 kernel converts int64 to double with the scalar cvtsi2sd instruction while SIMDe uses split arithmetic; add a regression test that decodes INT64_MIN/MAX and powers of two through both kernels and the scalar reference. Also finish the x86-64 wording in the benchmark build comment and design doc. --- cpp/bench_mark/alp_avx2_benchmark.cc | 6 ++-- cpp/docs/alp-codec-design.md | 2 +- cpp/test/encoding/alp_simde_test.cc | 49 ++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/cpp/bench_mark/alp_avx2_benchmark.cc b/cpp/bench_mark/alp_avx2_benchmark.cc index fa1a20c86..88ebc9e2e 100644 --- a/cpp/bench_mark/alp_avx2_benchmark.cc +++ b/cpp/bench_mark/alp_avx2_benchmark.cc @@ -21,8 +21,10 @@ // the runtime-dispatched native AVX2 copies. Build with the default flags // (no -mavx2) so the SIMDe side is what a portable build ships: // -// cmake -DBUILD_BENCHMARK=ON -DBUILD_TEST=OFF .. -// ./alp_avx2_benchmark +// cmake -S cpp -B cpp/build/Release \ +// -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARK=ON -DBUILD_TEST=OFF +// cmake --build cpp/build/Release --target alp_avx2_benchmark +// ./cpp/build/Release/alp_avx2_benchmark // // When ALP AVX2 dispatch is not compiled (non-x86-64 targets) or the CPU has no // AVX2, only the SIMDe column is measured. diff --git a/cpp/docs/alp-codec-design.md b/cpp/docs/alp-codec-design.md index 2f7503ed1..3026a1b6f 100644 --- a/cpp/docs/alp-codec-design.md +++ b/cpp/docs/alp-codec-design.md @@ -184,7 +184,7 @@ SIMDe is already a TsFile C++ dependency. The ALP kernels will use the existing - Constant blocks, single-value blocks, partial final blocks, and multi-block pages. - Scalar and SIMD must encode to identical bytes and decode to identical - values. On AVX2-capable x86 machines, differential tests also compare the + values. On AVX2-capable x86-64 machines, differential tests also compare the dispatched AVX2 kernels against the SIMDe kernels directly. - Corrupt/truncated payload tests must return errors without out-of-bounds access. diff --git a/cpp/test/encoding/alp_simde_test.cc b/cpp/test/encoding/alp_simde_test.cc index 515973fb8..920b60c06 100644 --- a/cpp/test/encoding/alp_simde_test.cc +++ b/cpp/test/encoding/alp_simde_test.cc @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include @@ -508,6 +509,54 @@ TEST(AlpAvx2Test, DecodeMalformedStatusMatchesSimde) { encoded, 4, 0, 0, bitmap, exceptions, 2, out)); } +// AVX2 has no packed int64 -> double conversion, so the AVX2 kernel uses a +// scalar conversion per lane while SIMDe uses a split-arithmetic trick. Check +// the values where that conversion is most delicate, against SIMDe and against +// the scalar reference (factor 0, exponent 0 decodes to (double)encoded). +TEST(AlpAvx2Test, DoubleDecodeAtInt64ExtremesMatchesReference) { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + typedef AlpTypeTraits Traits; + const int64_t kMin = std::numeric_limits::min(); + const int64_t kMax = std::numeric_limits::max(); + const int64_t encoded[16] = { + kMin, + kMin + 1, + -((int64_t)1 << 62), + -((int64_t)1 << 53), + -((int64_t)1 << 52) - 1, + -1, + 0, + 1, + ((int64_t)1 << 52) - 1, + (int64_t)1 << 52, + ((int64_t)1 << 52) + 1, + (int64_t)1 << 53, + ((int64_t)1 << 62) - 1, + (int64_t)1 << 62, + kMax - 1, + kMax, + }; + const uint32_t count = sizeof(encoded) / sizeof(encoded[0]); + std::vector simde_out(count); + std::vector avx2_out(count); + ASSERT_EQ(ALP_OK, + AlpSimdeKernel::DecodeValues( + encoded, count, 0, 0, nullptr, nullptr, 0, &simde_out[0])); + ASSERT_EQ(ALP_OK, + avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, count, 0, 0, nullptr, nullptr, 0, &avx2_out[0])); + for (uint32_t i = 0; i < count; ++i) { + EXPECT_TRUE(Traits::BitwiseEqual(simde_out[i], avx2_out[i])) + << "SIMDe mismatch at index " << i; + EXPECT_TRUE(Traits::BitwiseEqual(Traits::DecodeValue(encoded[i], 0, 0), + avx2_out[i])) + << "reference mismatch at index " << i; + } +} + #endif // ALP_AVX2_DISPATCH #endif // ENABLE_SIMD From e6e12ad153b814f6c3ac921309d6c0a3a6ae499a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 14:10:11 +0800 Subject: [PATCH 6/7] fix(cpp): address ALP AVX2 review feedback (round 3) Guard the benchmark speedup column against a zero SIMDe rate, copy the converted int64 lanes with std::memcpy instead of aliasing a __m256i store, and make the test encode helpers return a status so the caller can assert fatally instead of continuing after a failed encode. --- cpp/bench_mark/alp_avx2_benchmark.cc | 11 ++++-- cpp/src/encoding/alp/alp_avx2.h | 4 ++- cpp/test/encoding/alp_simde_test.cc | 51 ++++++++++++++-------------- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/cpp/bench_mark/alp_avx2_benchmark.cc b/cpp/bench_mark/alp_avx2_benchmark.cc index 88ebc9e2e..c39dbb048 100644 --- a/cpp/bench_mark/alp_avx2_benchmark.cc +++ b/cpp/bench_mark/alp_avx2_benchmark.cc @@ -168,10 +168,15 @@ void PrintRow(const std::string& pattern, const std::string& pair, << std::right << std::setw(16) << std::fixed << std::setprecision(2) << simde_mps; if (avx2_mps > 0.0) { - std::cout << std::setw(16) << avx2_mps << std::setw(12) - << std::setprecision(2) << (avx2_mps / simde_mps); + std::cout << std::setw(16) << avx2_mps; } else { - std::cout << std::setw(16) << "-" << std::setw(12) << "-"; + std::cout << std::setw(16) << "-"; + } + if (avx2_mps > 0.0 && simde_mps > 0.0) { + std::cout << std::setw(12) << std::setprecision(2) + << (avx2_mps / simde_mps); + } else { + std::cout << std::setw(12) << "-"; } std::cout << std::setw(10) << (ok ? "yes" : "no") << std::endl; } diff --git a/cpp/src/encoding/alp/alp_avx2.h b/cpp/src/encoding/alp/alp_avx2.h index 4f5044bb7..103103499 100644 --- a/cpp/src/encoding/alp/alp_avx2.h +++ b/cpp/src/encoding/alp/alp_avx2.h @@ -31,6 +31,8 @@ #include +#include + #include "alp_traits.h" #include "common/simd_dispatch.h" @@ -68,7 +70,7 @@ static inline __m256i AlpAvx2CvttpdEpi64(__m256d v) { TSFILE_TARGET_AVX2 static inline __m256d AlpAvx2Cvtepi64Pd(__m256i v) { alignas(32) int64_t lanes[4]; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(lanes), v); + std::memcpy(lanes, &v, sizeof(lanes)); return _mm256_setr_pd( static_cast(lanes[0]), static_cast(lanes[1]), static_cast(lanes[2]), static_cast(lanes[3])); diff --git a/cpp/test/encoding/alp_simde_test.cc b/cpp/test/encoding/alp_simde_test.cc index 920b60c06..7c5e3e662 100644 --- a/cpp/test/encoding/alp_simde_test.cc +++ b/cpp/test/encoding/alp_simde_test.cc @@ -248,41 +248,41 @@ TEST(AlpSimdeTest, UnpackMatchesReferenceAcrossBitWidths) { // by the scalar reference agreeing with both. template -static uint32_t EncodeWithSimde( +static bool EncodeWithSimde( const std::vector& values, uint8_t factor, uint8_t exponent, std::vector::Encoded>& encoded, - std::vector& bitmap, std::vector& exceptions) { + std::vector& bitmap, std::vector& exceptions, + uint32_t* exception_count) { const uint32_t count = static_cast(values.size()); encoded.assign(count, 0); bitmap.assign((count + 7) / 8, 0); exceptions.assign(count, T()); - uint32_t exception_count = 0; - EXPECT_EQ(ALP_OK, AlpSimdeKernel::EncodeValues( - values.empty() ? nullptr : &values[0], count, factor, - exponent, encoded.empty() ? nullptr : &encoded[0], - bitmap.empty() ? nullptr : &bitmap[0], - exceptions.empty() ? nullptr : &exceptions[0], - &exception_count)); - return exception_count; + *exception_count = 0; + return AlpSimdeKernel::EncodeValues( + values.empty() ? nullptr : &values[0], count, factor, exponent, + encoded.empty() ? nullptr : &encoded[0], + bitmap.empty() ? nullptr : &bitmap[0], + exceptions.empty() ? nullptr : &exceptions[0], + exception_count) == ALP_OK; } template -static uint32_t EncodeWithAvx2( +static bool EncodeWithAvx2( const std::vector& values, uint8_t factor, uint8_t exponent, std::vector::Encoded>& encoded, - std::vector& bitmap, std::vector& exceptions) { + std::vector& bitmap, std::vector& exceptions, + uint32_t* exception_count) { const uint32_t count = static_cast(values.size()); encoded.assign(count, 0); bitmap.assign((count + 7) / 8, 0); exceptions.assign(count, T()); - uint32_t exception_count = 0; - EXPECT_EQ(ALP_OK, avx2_detail::AlpAvx2Kernel::EncodeValues( - values.empty() ? nullptr : &values[0], count, factor, - exponent, encoded.empty() ? nullptr : &encoded[0], - bitmap.empty() ? nullptr : &bitmap[0], - exceptions.empty() ? nullptr : &exceptions[0], - &exception_count)); - return exception_count; + *exception_count = 0; + return avx2_detail::AlpAvx2Kernel::EncodeValues( + values.empty() ? nullptr : &values[0], count, factor, exponent, + encoded.empty() ? nullptr : &encoded[0], + bitmap.empty() ? nullptr : &bitmap[0], + exceptions.empty() ? nullptr : &exceptions[0], + exception_count) == ALP_OK; } template @@ -298,11 +298,12 @@ static void ExpectAvx2MatchesSimdeForPair(const std::vector& values, std::vector avx2_bitmap; std::vector avx2_exceptions; - const uint32_t simde_count = - EncodeWithSimde(values, factor, exponent, simde_encoded, simde_bitmap, - simde_exceptions); - const uint32_t avx2_count = EncodeWithAvx2( - values, factor, exponent, avx2_encoded, avx2_bitmap, avx2_exceptions); + uint32_t simde_count = 0; + uint32_t avx2_count = 0; + ASSERT_TRUE(EncodeWithSimde(values, factor, exponent, simde_encoded, + simde_bitmap, simde_exceptions, &simde_count)); + ASSERT_TRUE(EncodeWithAvx2(values, factor, exponent, avx2_encoded, + avx2_bitmap, avx2_exceptions, &avx2_count)); ASSERT_EQ(simde_count, avx2_count); EXPECT_EQ(simde_bitmap, avx2_bitmap); EXPECT_EQ(simde_encoded, avx2_encoded); From 3722b32fe7e3ca696ddbb70bfc2c5f085ac1501b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 16:11:13 +0800 Subject: [PATCH 7/7] fix(cpp): zero exception slots in ALP encode tails The scalar tails of the SIMDe and AVX2 encode kernels left encoded[i] untouched on the exception path while the vector body and the scalar reference write 0. Zero them for deterministic output, and assert it in the differential test by pre-filling the output buffers with a sentinel. --- cpp/src/encoding/alp/alp_avx2.h | 2 ++ cpp/src/encoding/alp/alp_simde.h | 2 ++ cpp/test/encoding/alp_simde_test.cc | 16 ++++++++++++++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cpp/src/encoding/alp/alp_avx2.h b/cpp/src/encoding/alp/alp_avx2.h index 103103499..01a4b4db8 100644 --- a/cpp/src/encoding/alp/alp_avx2.h +++ b/cpp/src/encoding/alp/alp_avx2.h @@ -152,6 +152,7 @@ struct AlpAvx2Kernel { Traits::DecodeValue(value, factor, exponent), values[i])) { bitmap[i >> 3] |= static_cast(1u << (i & 7u)); exceptions[ex_count++] = values[i]; + encoded[i] = 0; } else { encoded[i] = value; } @@ -270,6 +271,7 @@ struct AlpAvx2Kernel { Traits::DecodeValue(value, factor, exponent), values[i])) { bitmap[i >> 3] |= static_cast(1u << (i & 7u)); exceptions[ex_count++] = values[i]; + encoded[i] = 0; } else { encoded[i] = value; } diff --git a/cpp/src/encoding/alp/alp_simde.h b/cpp/src/encoding/alp/alp_simde.h index 4e4bbd1e4..e334bd24b 100644 --- a/cpp/src/encoding/alp/alp_simde.h +++ b/cpp/src/encoding/alp/alp_simde.h @@ -122,6 +122,7 @@ struct AlpSimdeKernel { Traits::DecodeValue(value, factor, exponent), values[i])) { bitmap[i >> 3] |= static_cast(1u << (i & 7u)); exceptions[ex_count++] = values[i]; + encoded[i] = 0; } else { encoded[i] = value; } @@ -245,6 +246,7 @@ struct AlpSimdeKernel { Traits::DecodeValue(value, factor, exponent), values[i])) { bitmap[i >> 3] |= static_cast(1u << (i & 7u)); exceptions[ex_count++] = values[i]; + encoded[i] = 0; } else { encoded[i] = value; } diff --git a/cpp/test/encoding/alp_simde_test.cc b/cpp/test/encoding/alp_simde_test.cc index 7c5e3e662..05f5b0e3e 100644 --- a/cpp/test/encoding/alp_simde_test.cc +++ b/cpp/test/encoding/alp_simde_test.cc @@ -253,8 +253,10 @@ static bool EncodeWithSimde( std::vector::Encoded>& encoded, std::vector& bitmap, std::vector& exceptions, uint32_t* exception_count) { + typedef typename AlpTypeTraits::Encoded Encoded; const uint32_t count = static_cast(values.size()); - encoded.assign(count, 0); + // A sentinel so a slot the kernel forgets to write is visible below. + encoded.assign(count, static_cast(0x5A5A5A5A)); bitmap.assign((count + 7) / 8, 0); exceptions.assign(count, T()); *exception_count = 0; @@ -272,8 +274,9 @@ static bool EncodeWithAvx2( std::vector::Encoded>& encoded, std::vector& bitmap, std::vector& exceptions, uint32_t* exception_count) { + typedef typename AlpTypeTraits::Encoded Encoded; const uint32_t count = static_cast(values.size()); - encoded.assign(count, 0); + encoded.assign(count, static_cast(0x5A5A5A5A)); bitmap.assign((count + 7) / 8, 0); exceptions.assign(count, T()); *exception_count = 0; @@ -307,6 +310,15 @@ static void ExpectAvx2MatchesSimdeForPair(const std::vector& values, ASSERT_EQ(simde_count, avx2_count); EXPECT_EQ(simde_bitmap, avx2_bitmap); EXPECT_EQ(simde_encoded, avx2_encoded); + // Exception slots must be written deterministically even when the caller + // does not pre-zero the output (the buffers start at a sentinel). + for (uint32_t i = 0; i < static_cast(values.size()); ++i) { + if ((simde_bitmap[i >> 3] & static_cast(1u << (i & 7u))) != + 0) { + EXPECT_EQ(0, simde_encoded[i]) << "SIMDe exception slot " << i; + EXPECT_EQ(0, avx2_encoded[i]) << "AVX2 exception slot " << i; + } + } for (uint32_t i = 0; i < simde_count; ++i) { EXPECT_TRUE( Traits::BitwiseEqual(simde_exceptions[i], avx2_exceptions[i]))