diff --git a/cmake/multi-arch.cmake b/cmake/multi-arch.cmake index aeb81e693..29875ceb8 100644 --- a/cmake/multi-arch.cmake +++ b/cmake/multi-arch.cmake @@ -16,6 +16,8 @@ set(SVS_X86_SRC_DIR "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86") set(SVS_X86 "${SVS_X86_SRC_DIR}/avx2.cpp,avx2,haswell" "${SVS_X86_SRC_DIR}/avx512.cpp,avx512,cascadelake" + "${SVS_X86_SRC_DIR}/avx512_fp16.cpp,avx512fp16,cascadelake" + ) set(SVS_X86_OBJECT_FILES) @@ -26,8 +28,21 @@ foreach(x86_info IN LISTS SVS_X86) list(GET x86_info 2 arch) set(lib_name "svs_x86_${avx}") add_library(${lib_name} INTERFACE) + # Base arch tuning target_compile_options(${lib_name} INTERFACE -march=${arch} -mtune=${arch}) - set(obj_name ${arch}_obj) + # If building AVX512-FP16 variant, request the specific FP16 ISA flags if supported + if("${avx}" STREQUAL "avx512fp16") + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-mavx512fp16" SVS_COMPILER_HAS_AVX512FP16) + check_cxx_compiler_flag("-mavx512vl" SVS_COMPILER_HAS_AVX512VL) + if(SVS_COMPILER_HAS_AVX512FP16 AND SVS_COMPILER_HAS_AVX512VL) + target_compile_options(${lib_name} INTERFACE -mavx512fp16 -mavx512vl) + else() + message(STATUS "Compiler does not support -mavx512fp16/-mavx512vl; building FP16 object without those flags for validation only.") + endif() + endif() + # Ensure object target names are unique per arch+avx variant + set(obj_name ${arch}_${avx}_obj) add_library(${obj_name} OBJECT ${src}) target_link_libraries(${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name}) diff --git a/include/svs/core/distance/cosine.h b/include/svs/core/distance/cosine.h index 9f4924997..5339edbad 100644 --- a/include/svs/core/distance/cosine.h +++ b/include/svs/core/distance/cosine.h @@ -383,6 +383,11 @@ struct CosineSimilarityImpl { #endif +// Everything not natively overridden in avx512_fp16.cpp reuses the AVX512 implementation. +template +struct CosineSimilarityImpl + : CosineSimilarityImpl {}; + ///// ///// Intel(R) AVX2 Implementations ///// diff --git a/include/svs/core/distance/distance_core.h b/include/svs/core/distance/distance_core.h index 4f59f9de0..9724be696 100644 --- a/include/svs/core/distance/distance_core.h +++ b/include/svs/core/distance/distance_core.h @@ -26,7 +26,7 @@ namespace svs::distance { -enum class AVX_AVAILABILITY { NONE, AVX2, AVX512 }; +enum class AVX_AVAILABILITY { NONE, AVX2, AVX512, AVX512_FP16 }; constexpr std::array supported_dim_list{ 64, 96, 100, 128, 160, 200, 512, 768, svs::Dynamic}; diff --git a/include/svs/core/distance/euclidean.h b/include/svs/core/distance/euclidean.h index b038a6fcc..97807914a 100644 --- a/include/svs/core/distance/euclidean.h +++ b/include/svs/core/distance/euclidean.h @@ -86,6 +86,11 @@ class L2 { public: template static constexpr float compute(const Ea* a, const Eb* b, size_t N) { + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512fp16_supported(), 1)) { + return L2Impl::compute( + a, b, lib::MaybeStatic(N) + ); + } if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { return L2Impl::compute( a, b, lib::MaybeStatic(N) @@ -103,6 +108,17 @@ class L2 { template static constexpr float compute(const Ea* a, const Eb* b) { + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512fp16_supported(), 1)) { + if constexpr (is_dim_supported()) { + return L2Impl::compute( + a, b, lib::MaybeStatic() + ); + } else { + return L2Impl::compute( + a, b, lib::MaybeStatic(N) + ); + } + } if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { if constexpr (is_dim_supported()) { return L2Impl::compute( @@ -359,6 +375,11 @@ template struct L2Impl #endif +// Everything not natively overridden in avx512_fp16.cpp reuses the AVX512 implementation. +template +struct L2Impl + : L2Impl {}; + ///// ///// Intel(R) AVX2 Implementations ///// diff --git a/include/svs/core/distance/inner_product.h b/include/svs/core/distance/inner_product.h index 0f7837a53..95f59cac1 100644 --- a/include/svs/core/distance/inner_product.h +++ b/include/svs/core/distance/inner_product.h @@ -46,6 +46,11 @@ class IP { public: template static constexpr float compute(const Ea* a, const Eb* b, size_t N) { + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512fp16_supported(), 1)) { + return IPImpl::compute( + a, b, lib::MaybeStatic(N) + ); + } if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { return IPImpl::compute( a, b, lib::MaybeStatic(N) @@ -63,6 +68,17 @@ class IP { template static constexpr float compute(const Ea* a, const Eb* b) { + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512fp16_supported(), 1)) { + if constexpr (is_dim_supported()) { + return IPImpl::compute( + a, b, lib::MaybeStatic() + ); + } else { + return IPImpl::compute( + a, b, lib::MaybeStatic(N) + ); + } + } if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { if constexpr (is_dim_supported()) { return IPImpl::compute( @@ -312,6 +328,31 @@ template struct IPImpl }; #endif +// Everything not natively overridden in avx512_fp16.cpp reuses the AVX512 implementation. +template +struct IPImpl + : IPImpl {}; + +// Explicit partial specialization for Float16×Float16 under AVX512_FP16. +// The struct is declared here; the member function is defined out-of-line in +// avx512_fp16.cpp so the native vfmaddph kernel is linked into every consumer TU. +template +struct IPImpl { + SVS_NOINLINE static float compute( + const Float16* a, const Float16* b, lib::MaybeStatic length + ); +}; + +// Explicit partial specialization for float (query) × Float16 (data) under AVX512_FP16. +// Converts the float32 query to fp16 on the fly and uses the native vfmaddph path. +// The member function is defined out-of-line in avx512_fp16.cpp. +template +struct IPImpl { + SVS_NOINLINE static float compute( + const float* a, const Float16* b, lib::MaybeStatic length + ); +}; + ///// ///// Intel(R) AVX2 Implementations ///// @@ -408,6 +449,16 @@ DISTANCE_IP_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); DISTANCE_IP_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX2); DISTANCE_IP_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX2); DISTANCE_IP_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); + +DISTANCE_IP_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512_FP16); #endif } // namespace svs::distance diff --git a/include/svs/lib/avx_detection.h b/include/svs/lib/avx_detection.h index fc9b246f9..3df8fb4df 100644 --- a/include/svs/lib/avx_detection.h +++ b/include/svs/lib/avx_detection.h @@ -36,15 +36,19 @@ struct AVXRuntimeFlags { avx2 = extended_features && ((ebx & (1 << 5)) != 0); avx512f = extended_features && ((ebx & (1 << 16)) != 0); avx512vnni = extended_features && ((ecx & (1 << 11)) != 0); + // AVX512-FP16 reported in CPUID leaf 7 subleaf 0 EDX bit 23 (Intel SDM vol.2 Table 3-8) + avx512fp16 = extended_features && ((edx & (1u << 23)) != 0); } bool is_avx2_supported() const noexcept { return avx2; } bool is_avx512f_supported() const noexcept { return avx512f; } bool is_avx512vnni_supported() const noexcept { return avx512vnni; } + bool is_avx512fp16_supported() const noexcept { return avx512fp16; } bool avx2; bool avx512f; bool avx512vnni; + bool avx512fp16; }; #else struct AVXRuntimeFlags { diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index bee150d75..62b4e8b38 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -41,6 +41,7 @@ DISTANCE_IP_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512); DISTANCE_IP_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); DISTANCE_IP_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); DISTANCE_IP_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); +// Keep existing AVX512 instantiation; if FP16-capable runtime exists prefer that path. DISTANCE_CS_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512); DISTANCE_CS_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512); diff --git a/include/svs/multi-arch/x86/avx512_fp16.cpp b/include/svs/multi-arch/x86/avx512_fp16.cpp new file mode 100644 index 000000000..6b22fd7b7 --- /dev/null +++ b/include/svs/multi-arch/x86/avx512_fp16.cpp @@ -0,0 +1,181 @@ +#if defined(__x86_64__) +#include "svs/core/distance/cosine.h" +#include "svs/core/distance/euclidean.h" +#include "svs/core/distance/inner_product.h" + +// This translation unit is compiled with -mavx512fp16 -mavx512vl by cmake to +// validate compilation of native fp16 intrinsics and provide an experimental +// native-FP16 implementation for Float16 x Float16 inner product. The +// implementation intentionally stays small and uses the existing SIMD helper +// infrastructure. When the compiler/CPU lacks FP16 support this TU still +// compiles (the CMake logic will omit the flags) and the code falls back to +// the AVX512 fp32-accumulating path. + +namespace svs::distance { + +///// +///// Native AVX512-FP16 kernel for Float16 x Float16 inner product. +///// +// IPNativeHalfOp32 uses the _Float16 / __m512h intrinsics introduced in +// Intel's AVX512-FP16 ISA extension (Sapphire Rapids and later). Each 512-bit +// register holds 32 fp16 elements, so simd_width=32. The masked load is +// implemented via _mm512_maskz_loadu_epi16 (zero-masking) cast to __m512h, +// which avoids the need for a separate blend and keeps inactive lanes at zero. +// +// Guard: __AVX512FP16__ is defined by GCC/Clang when -mavx512fp16 is passed. +// We do NOT use SVS_AVX512_F here because that only tests for AVX512F (present +// on Skylake) whereas FP16 native arithmetic requires Sapphire Rapids+. + +#if defined(__AVX512FP16__) +struct IPNativeHalfOp32 { + static constexpr size_t simd_width = 32; + using mask_t = svs::mask_repr_t<32>; // uint32_t + + static __m512h init() { return _mm512_setzero_ph(); } + + static __m512h load_a(const Float16* p) { + return _mm512_loadu_ph(reinterpret_cast(p)); + } + static __m512h load_a(mask_t m, const Float16* p) { + return _mm512_castsi512_ph( + _mm512_maskz_loadu_epi16((__mmask32)m, reinterpret_cast(p))); + } + static __m512h load_b(const Float16* p) { return load_a(p); } + static __m512h load_b(mask_t m, const Float16* p) { return load_a(m, p); } + + static __m512h accumulate(__m512h acc, __m512h a, __m512h b) { + return _mm512_fmadd_ph(a, b, acc); + } + static __m512h accumulate(mask_t m, __m512h acc, __m512h a, __m512h b) { + // Masked FMA: result[i] = mask[i] ? a[i]*b[i]+acc[i] : acc[i] + return _mm512_mask_add_ph(acc, (__mmask32)m, acc, _mm512_mul_ph(a, b)); + } + static __m512h combine(__m512h x, __m512h y) { return _mm512_add_ph(x, y); } + static float reduce(__m512h x) { + return static_cast(_mm512_reduce_add_ph(x)); + } +}; +#endif // __AVX512FP16__ + +///// +///// Native AVX512-FP16 kernel for float (query) x Float16 (data) inner product. +///// +// IPMixedFloatToHalfOp32 converts the float32 query to fp16 on the fly and then +// performs the same vfmaddph computation as IPNativeHalfOp32. +// load_a handles the float query (needs down-conversion to fp16). +// load_b handles the Float16 data (loaded directly). + +#if defined(__AVX512FP16__) +struct IPMixedFloatToHalfOp32 { + static constexpr size_t simd_width = 32; + using mask_t = svs::mask_repr_t<32>; // uint32_t + + static __m512h init() { return _mm512_setzero_ph(); } + + // Query is float32 — load 32 floats (2×__m512), convert to __m512h + static __m512h load_a(const float* p) { + __m256i lo = _mm512_cvtps_ph(_mm512_loadu_ps(p), _MM_FROUND_NO_EXC); + __m256i hi = _mm512_cvtps_ph(_mm512_loadu_ps(p + 16), _MM_FROUND_NO_EXC); + return _mm512_castsi512_ph( + _mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1)); + } + static __m512h load_a(mask_t m, const float* p) { + // For the ragged tail — load up to 32 floats masked. + // Split the 32-bit mask into two 16-bit halves for the two __m512 loads. + __mmask16 m_lo = (__mmask16)(m & 0xFFFF); + __mmask16 m_hi = (__mmask16)(m >> 16); + __m256i lo = _mm512_cvtps_ph( + _mm512_maskz_loadu_ps(m_lo, p), _MM_FROUND_NO_EXC); + __m256i hi = _mm512_cvtps_ph( + _mm512_maskz_loadu_ps(m_hi, p + 16), _MM_FROUND_NO_EXC); + return _mm512_castsi512_ph( + _mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1)); + } + + // Data is Float16 — load directly + static __m512h load_b(const Float16* p) { + return _mm512_loadu_ph(reinterpret_cast(p)); + } + static __m512h load_b(mask_t m, const Float16* p) { + return _mm512_castsi512_ph( + _mm512_maskz_loadu_epi16((__mmask32)m, reinterpret_cast(p))); + } + + static __m512h accumulate(__m512h acc, __m512h a, __m512h b) { + return _mm512_fmadd_ph(a, b, acc); + } + static __m512h accumulate(mask_t m, __m512h acc, __m512h a, __m512h b) { + return _mm512_mask_add_ph(acc, (__mmask32)m, acc, _mm512_mul_ph(a, b)); + } + static __m512h combine(__m512h x, __m512h y) { return _mm512_add_ph(x, y); } + static float reduce(__m512h x) { + return static_cast(_mm512_reduce_add_ph(x)); + } +}; +#endif // __AVX512FP16__ + +// Out-of-line definition of IPImpl::compute. +// The struct is declared in inner_product.h so every consumer TU resolves to this +// native kernel rather than the catch-all inheritance fallback. +// When __AVX512FP16__ is defined (gcc/clang with -mavx512fp16), we use the +// native vfmaddph instruction path. Otherwise we fall back to the AVX512 +// fp32-accumulating path. +template +SVS_NOINLINE float IPImpl::compute( + const Float16* a, const Float16* b, lib::MaybeStatic length +) { +#if defined(__AVX512FP16__) + return svs::simd::generic_simd_op(IPNativeHalfOp32{}, a, b, length); +#else + return IPImpl::compute(a, b, length); +#endif +} + +// Out-of-line definition of IPImpl::compute. +// Converts the float32 query to fp16 on the fly, then uses vfmaddph. +template +SVS_NOINLINE float IPImpl::compute( + const float* a, const Float16* b, lib::MaybeStatic length +) { +#if defined(__AVX512FP16__) + return svs::simd::generic_simd_op(IPMixedFloatToHalfOp32{}, a, b, length); +#else + return IPImpl::compute(a, b, length); +#endif +} + +// Reuse the AVX512 instantiations but under a distinct availability tag so +// runtime selection can prefer FP16-capable code paths. +DISTANCE_L2_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_L2_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512_FP16); + +DISTANCE_IP_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_IP_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512_FP16); + +DISTANCE_CS_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512_FP16); +DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512_FP16); + +} // namespace svs::distance + +#endif