From c724d32bc1e68c33e680bbe566bd9e1c1e38ad3a Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Wed, 22 Jul 2026 16:07:41 +0300 Subject: [PATCH] Fix unaligned distance metadata loads [MOD-15303] --- src/VecSim/spaces/IP/IP.cpp | 34 +++++----- src/VecSim/spaces/IP/IP_AVX2_FMA_SQ8_FP32.h | 6 +- src/VecSim/spaces/IP/IP_AVX2_SQ8_FP32.h | 6 +- .../spaces/IP/IP_AVX512F_BW_VL_VNNI_INT8.h | 6 +- .../IP/IP_AVX512F_BW_VL_VNNI_SQ8_FP32.h | 6 +- .../spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h | 16 ++--- .../spaces/IP/IP_AVX512F_BW_VL_VNNI_UINT8.h | 6 +- src/VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h | 6 +- .../spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h | 16 ++--- src/VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h | 6 +- src/VecSim/spaces/IP/IP_NEON_INT8.h | 6 +- src/VecSim/spaces/IP/IP_NEON_SQ8_FP32.h | 6 +- src/VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h | 16 ++--- src/VecSim/spaces/IP/IP_NEON_UINT8.h | 6 +- src/VecSim/spaces/IP/IP_SSE4_SQ8_FP32.h | 8 +-- src/VecSim/spaces/IP/IP_SVE_INT8.h | 6 +- src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h | 6 +- src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h | 16 ++--- src/VecSim/spaces/IP/IP_SVE_UINT8.h | 6 +- src/VecSim/spaces/L2/L2.cpp | 10 +-- src/VecSim/spaces/L2/L2_AVX2_FMA_SQ8_FP32.h | 4 +- src/VecSim/spaces/L2/L2_AVX2_SQ8_FP32.h | 4 +- .../L2/L2_AVX512F_BW_VL_VNNI_SQ8_FP32.h | 4 +- .../spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_SQ8.h | 6 +- .../spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h | 4 +- src/VecSim/spaces/L2/L2_NEON_SQ8_FP32.h | 4 +- src/VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h | 4 +- src/VecSim/spaces/L2/L2_SSE4_SQ8_FP32.h | 4 +- src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h | 4 +- src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h | 4 +- src/VecSim/spaces/normalize/normalize_naive.h | 5 +- .../spaces_benchmarks/bm_spaces_sq8_fp32.cpp | 18 ++--- tests/unit/test_components.cpp | 15 +++-- tests/unit/test_index_test_utils.cpp | 4 +- tests/unit/test_int8.cpp | 4 +- tests/unit/test_spaces.cpp | 66 ++++++++++++++++--- tests/unit/test_uint8.cpp | 2 +- tests/utils/tests_utils.h | 34 +++++----- 38 files changed, 210 insertions(+), 174 deletions(-) diff --git a/src/VecSim/spaces/IP/IP.cpp b/src/VecSim/spaces/IP/IP.cpp index a3786a8f1..2140c2345 100644 --- a/src/VecSim/spaces/IP/IP.cpp +++ b/src/VecSim/spaces/IP/IP.cpp @@ -56,10 +56,10 @@ float SQ8_FP32_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, size_ // Combine accumulators float quantized_dot = (sum0 + sum1) + (sum2 + sum3); - // Get quantization parameters from stored vector (pVect1 is SQ8) - const float *params = reinterpret_cast(pVect1 + dimension); - const float min_val = params[sq8::MIN_VAL]; - const float delta = params[sq8::DELTA]; + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *params = pVect1 + dimension; + const float min_val = load_unaligned(params + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + sq8::DELTA * sizeof(float)); // Get precomputed y_sum from query blob (pVect2 is FP32, stored after the dim floats) const float y_sum = pVect2[dimension + sq8::SUM_QUERY]; @@ -152,17 +152,17 @@ float SQ8_SQ8_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, size_t product += pVect1[i] * pVect2[i]; } - // Get quantization parameters from pVect1 - const float *params1 = reinterpret_cast(pVect1 + dimension); - const float min_val1 = params1[sq8::MIN_VAL]; - const float delta1 = params1[sq8::DELTA]; - const float sum1 = params1[sq8::SUM]; + // Metadata follows byte payloads and is not necessarily float-aligned. + const auto *params1 = pVect1 + dimension; + const float min_val1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); // Get quantization parameters from pVect2 - const float *params2 = reinterpret_cast(pVect2 + dimension); - const float min_val2 = params2[sq8::MIN_VAL]; - const float delta2 = params2[sq8::DELTA]; - const float sum2 = params2[sq8::SUM]; + const auto *params2 = pVect2 + dimension; + const float min_val2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); // Apply the algebraic formula using precomputed sums: // IP = min1*sum2 + min2*sum1 + delta1*delta2*Σ(q1[i]*q2[i]) - dim*min1*min2 @@ -265,8 +265,8 @@ float INT8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension) { const auto *pVect1 = static_cast(pVect1v); const auto *pVect2 = static_cast(pVect2v); // We expect the vectors' norm to be stored at the end of the vector. - float norm_v1 = *reinterpret_cast(pVect1 + dimension); - float norm_v2 = *reinterpret_cast(pVect2 + dimension); + const float norm_v1 = load_unaligned(pVect1 + dimension); + const float norm_v2 = load_unaligned(pVect2 + dimension); return 1.0f - float(INTEGER_InnerProductImp(pVect1, pVect2, dimension)) / (norm_v1 * norm_v2); } @@ -280,7 +280,7 @@ float UINT8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension) { const auto *pVect1 = static_cast(pVect1v); const auto *pVect2 = static_cast(pVect2v); // We expect the vectors' norm to be stored at the end of the vector. - float norm_v1 = *reinterpret_cast(pVect1 + dimension); - float norm_v2 = *reinterpret_cast(pVect2 + dimension); + const float norm_v1 = load_unaligned(pVect1 + dimension); + const float norm_v2 = load_unaligned(pVect2 + dimension); return 1.0f - float(INTEGER_InnerProductImp(pVect1, pVect2, dimension)) / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_AVX2_FMA_SQ8_FP32.h b/src/VecSim/spaces/IP/IP_AVX2_FMA_SQ8_FP32.h index 8aebf768c..ecfd205e3 100644 --- a/src/VecSim/spaces/IP/IP_AVX2_FMA_SQ8_FP32.h +++ b/src/VecSim/spaces/IP/IP_AVX2_FMA_SQ8_FP32.h @@ -106,9 +106,9 @@ float SQ8_FP32_InnerProductImp_FMA(const void *pVect1v, const void *pVect2v, siz // Get quantization parameters from stored vector (after quantized data) const uint8_t *pVect1Base = static_cast(pVect1v); - const float *params1 = reinterpret_cast(pVect1Base + dimension); - const float min_val = params1[sq8::MIN_VAL]; - const float delta = params1[sq8::DELTA]; + const auto *params1 = pVect1Base + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); // Get precomputed y_sum from query blob (stored after the dim floats) const float y_sum = static_cast(pVect2v)[dimension + sq8::SUM_QUERY]; diff --git a/src/VecSim/spaces/IP/IP_AVX2_SQ8_FP32.h b/src/VecSim/spaces/IP/IP_AVX2_SQ8_FP32.h index 5152e4544..304030ffc 100644 --- a/src/VecSim/spaces/IP/IP_AVX2_SQ8_FP32.h +++ b/src/VecSim/spaces/IP/IP_AVX2_SQ8_FP32.h @@ -106,9 +106,9 @@ float SQ8_FP32_InnerProductImp_AVX2(const void *pVect1v, const void *pVect2v, si // Get quantization parameters from stored vector (after quantized data) const uint8_t *pVect1Base = static_cast(pVect1v); - const float *params1 = reinterpret_cast(pVect1Base + dimension); - const float min_val = params1[sq8::MIN_VAL]; - const float delta = params1[sq8::DELTA]; + const auto *params1 = pVect1Base + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); // Get precomputed y_sum from query blob (stored after the dim floats) const float y_sum = static_cast(pVect2v)[dimension + sq8::SUM_QUERY]; diff --git a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_INT8.h b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_INT8.h index add070942..6563bd034 100644 --- a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_INT8.h +++ b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_INT8.h @@ -71,9 +71,7 @@ template // 0..63 float INT8_CosineSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = INT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_FP32.h b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_FP32.h index c2048d01f..da5805150 100644 --- a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_FP32.h +++ b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_FP32.h @@ -97,9 +97,9 @@ float SQ8_FP32_InnerProductImp_AVX512(const void *pVec1v, const void *pVec2v, si // Get quantization parameters from stored vector (after quantized data) // Use the original base pointer since pVec1 has been advanced const uint8_t *pVec1Base = static_cast(pVec1v); - const float *params1 = reinterpret_cast(pVec1Base + dimension); - const float min_val = params1[sq8::MIN_VAL]; - const float delta = params1[sq8::DELTA]; + const auto *params1 = pVec1Base + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); // Get precomputed y_sum from query blob (stored after the dim floats) // Use the original base pointer since pVec2 has been advanced diff --git a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h index 899b466b9..ae6f96ea2 100644 --- a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h +++ b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h @@ -47,15 +47,15 @@ float SQ8_SQ8_InnerProductImp(const void *pVec1v, const void *pVec2v, size_t dim const uint8_t *pVec1 = static_cast(pVec1v); const uint8_t *pVec2 = static_cast(pVec2v); - const float *params1 = reinterpret_cast(pVec1 + dimension); - const float min1 = params1[sq8::MIN_VAL]; - const float delta1 = params1[sq8::DELTA]; - const float sum1 = params1[sq8::SUM]; // Precomputed sum of original float elements + const auto *params1 = pVec1 + dimension; + const float min1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); - const float *params2 = reinterpret_cast(pVec2 + dimension); - const float min2 = params2[sq8::MIN_VAL]; - const float delta2 = params2[sq8::DELTA]; - const float sum2 = params2[sq8::SUM]; // Precomputed sum of original float elements + const auto *params2 = pVec2 + dimension; + const float min2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); // Apply the algebraic formula using precomputed sums: // IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1[i]*q2[i]) - dim*min1*min2 diff --git a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_UINT8.h b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_UINT8.h index deed0f706..bd43bc901 100644 --- a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_UINT8.h +++ b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_UINT8.h @@ -100,9 +100,7 @@ template // 0..63 float UINT8_CosineSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = UINT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h b/src/VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h index 9fbc2b28d..13bac7ac4 100644 --- a/src/VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h +++ b/src/VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h @@ -113,9 +113,7 @@ float INT8_InnerProductSIMD16_NEON_DOTPROD(const void *pVect1v, const void *pVec template // 0..63 float INT8_CosineSIMD_NEON_DOTPROD(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = INT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h b/src/VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h index 7a122974f..d7f5b444e 100644 --- a/src/VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h +++ b/src/VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h @@ -48,15 +48,15 @@ float SQ8_SQ8_InnerProductSIMD64_NEON_DOTPROD_IMP(const void *pVec1v, const void const uint8_t *pVec1 = static_cast(pVec1v); const uint8_t *pVec2 = static_cast(pVec2v); - const float *params1 = reinterpret_cast(pVec1 + dimension); - const float min1 = params1[sq8::MIN_VAL]; - const float delta1 = params1[sq8::DELTA]; - const float sum1 = params1[sq8::SUM]; // Precomputed sum of original float elements + const auto *params1 = pVec1 + dimension; + const float min1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); - const float *params2 = reinterpret_cast(pVec2 + dimension); - const float min2 = params2[sq8::MIN_VAL]; - const float delta2 = params2[sq8::DELTA]; - const float sum2 = params2[sq8::SUM]; // Precomputed sum of original float elements + const auto *params2 = pVec2 + dimension; + const float min2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); // Apply algebraic formula using precomputed sums: // IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1*q2) - dim*min1*min2 diff --git a/src/VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h b/src/VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h index 73682a21a..3abbd9bba 100644 --- a/src/VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h +++ b/src/VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h @@ -111,9 +111,7 @@ float UINT8_InnerProductSIMD16_NEON_DOTPROD(const void *pVect1v, const void *pVe template // 0..63 float UINT8_CosineSIMD_NEON_DOTPROD(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = UINT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_NEON_INT8.h b/src/VecSim/spaces/IP/IP_NEON_INT8.h index 5118908d6..234454a36 100644 --- a/src/VecSim/spaces/IP/IP_NEON_INT8.h +++ b/src/VecSim/spaces/IP/IP_NEON_INT8.h @@ -119,9 +119,7 @@ float INT8_InnerProductSIMD16_NEON(const void *pVect1v, const void *pVect2v, siz template // 0..63 float INT8_CosineSIMD_NEON(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = INT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_NEON_SQ8_FP32.h b/src/VecSim/spaces/IP/IP_NEON_SQ8_FP32.h index 53a89bc7d..b4e27e491 100644 --- a/src/VecSim/spaces/IP/IP_NEON_SQ8_FP32.h +++ b/src/VecSim/spaces/IP/IP_NEON_SQ8_FP32.h @@ -113,9 +113,9 @@ float SQ8_FP32_InnerProductSIMD16_NEON_IMP(const void *pVect1v, const void *pVec // Get quantization parameters from stored vector (after quantized data) const uint8_t *pVect1Base = static_cast(pVect1v); - const float *params1 = reinterpret_cast(pVect1Base + dimension); - const float min_val = params1[sq8::MIN_VAL]; - const float delta = params1[sq8::DELTA]; + const auto *params1 = pVect1Base + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); // Get precomputed y_sum from query blob (stored after the dim floats) const float y_sum = static_cast(pVect2v)[dimension + sq8::SUM_QUERY]; diff --git a/src/VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h b/src/VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h index b89586322..3e931ee0e 100644 --- a/src/VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h +++ b/src/VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h @@ -48,16 +48,16 @@ float SQ8_SQ8_InnerProductSIMD64_NEON_IMP(const void *pVec1v, const void *pVec2v const uint8_t *pVec1 = static_cast(pVec1v); const uint8_t *pVec2 = static_cast(pVec2v); - const float *params1 = reinterpret_cast(pVec1 + dimension); - const float min1 = params1[sq8::MIN_VAL]; - const float delta1 = params1[sq8::DELTA]; - const float sum1 = params1[sq8::SUM]; // Precomputed sum of original float elements + const auto *params1 = pVec1 + dimension; + const float min1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); // Get dequantization parameters and precomputed values from the end of pVec2 - const float *params2 = reinterpret_cast(pVec2 + dimension); - const float min2 = params2[sq8::MIN_VAL]; - const float delta2 = params2[sq8::DELTA]; - const float sum2 = params2[sq8::SUM]; // Precomputed sum of original float elements + const auto *params2 = pVec2 + dimension; + const float min2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); // Apply algebraic formula using precomputed sums: // IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1*q2) - dim*min1*min2 diff --git a/src/VecSim/spaces/IP/IP_NEON_UINT8.h b/src/VecSim/spaces/IP/IP_NEON_UINT8.h index 6263eeea4..2d2b3f555 100644 --- a/src/VecSim/spaces/IP/IP_NEON_UINT8.h +++ b/src/VecSim/spaces/IP/IP_NEON_UINT8.h @@ -119,9 +119,7 @@ float UINT8_InnerProductSIMD16_NEON(const void *pVect1v, const void *pVect2v, si template // 0..63 float UINT8_CosineSIMD_NEON(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = UINT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_SSE4_SQ8_FP32.h b/src/VecSim/spaces/IP/IP_SSE4_SQ8_FP32.h index c71be8bdb..1eeaeb03a 100644 --- a/src/VecSim/spaces/IP/IP_SSE4_SQ8_FP32.h +++ b/src/VecSim/spaces/IP/IP_SSE4_SQ8_FP32.h @@ -28,7 +28,7 @@ using sq8 = vecsim_types::sq8; static inline void InnerProductStepSQ8_FP32(const uint8_t *&pVect1, const float *&pVect2, __m128 &sum) { // Load 4 uint8 elements and convert to float - __m128i v1_i = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(*reinterpret_cast(pVect1))); + __m128i v1_i = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(load_unaligned(pVect1))); pVect1 += 4; __m128 v1_f = _mm_cvtepi32_ps(v1_i); @@ -111,9 +111,9 @@ float SQ8_FP32_InnerProductSIMD16_SSE4_IMP(const void *pVect1v, const void *pVec // Get quantization parameters from stored vector (after quantized data) const uint8_t *pVect1Base = static_cast(pVect1v); - const float *params1 = reinterpret_cast(pVect1Base + dimension); - const float min_val = params1[sq8::MIN_VAL]; - const float delta = params1[sq8::DELTA]; + const auto *params1 = pVect1Base + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); // Get precomputed y_sum from query blob (stored after the dim floats) const float *pVect2Base = static_cast(pVect2v); diff --git a/src/VecSim/spaces/IP/IP_SVE_INT8.h b/src/VecSim/spaces/IP/IP_SVE_INT8.h index e8110bcff..5e58d6569 100644 --- a/src/VecSim/spaces/IP/IP_SVE_INT8.h +++ b/src/VecSim/spaces/IP/IP_SVE_INT8.h @@ -98,9 +98,7 @@ float INT8_InnerProductSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t template float INT8_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = INT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h b/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h index c4d5dbd7f..07e47a07e 100644 --- a/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h +++ b/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h @@ -120,9 +120,9 @@ float SQ8_FP32_InnerProductSIMD_SVE_IMP(const void *pVect1v, const void *pVect2v float quantized_dot = svaddv_f32(pg, sum); // Get quantization parameters from stored vector (after quantized data) - const float *params1 = reinterpret_cast(pVect1 + dimension); - const float min_val = params1[sq8::MIN_VAL]; - const float delta = params1[sq8::DELTA]; + const auto *params1 = pVect1 + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); // Get precomputed y_sum from query blob (stored after the dim floats) const float y_sum = pVect2[dimension + sq8::SUM_QUERY]; diff --git a/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h b/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h index a752817dd..93ddb76cb 100644 --- a/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h +++ b/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h @@ -48,15 +48,15 @@ float SQ8_SQ8_InnerProductSIMD_SVE_IMP(const void *pVec1v, const void *pVec2v, s const uint8_t *pVec1 = static_cast(pVec1v); const uint8_t *pVec2 = static_cast(pVec2v); - const float *params1 = reinterpret_cast(pVec1 + dimension); - const float min1 = params1[sq8::MIN_VAL]; - const float delta1 = params1[sq8::DELTA]; - const float sum1 = params1[sq8::SUM]; // Precomputed sum of original float elements + const auto *params1 = pVec1 + dimension; + const float min1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); - const float *params2 = reinterpret_cast(pVec2 + dimension); - const float min2 = params2[sq8::MIN_VAL]; - const float delta2 = params2[sq8::DELTA]; - const float sum2 = params2[sq8::SUM]; // Precomputed sum of original float elements + const auto *params2 = pVec2 + dimension; + const float min2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); // Apply algebraic formula with float conversion only at the end: // IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1*q2) - dim*min1*min2 diff --git a/src/VecSim/spaces/IP/IP_SVE_UINT8.h b/src/VecSim/spaces/IP/IP_SVE_UINT8.h index c1cc45b66..f6c6af3b8 100644 --- a/src/VecSim/spaces/IP/IP_SVE_UINT8.h +++ b/src/VecSim/spaces/IP/IP_SVE_UINT8.h @@ -97,9 +97,7 @@ float UINT8_InnerProductSIMD_SVE(const void *pVect1v, const void *pVect2v, size_ template float UINT8_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimension) { float ip = UINT8_InnerProductImp(pVect1v, pVect2v, dimension); - float norm_v1 = - *reinterpret_cast(static_cast(pVect1v) + dimension); - float norm_v2 = - *reinterpret_cast(static_cast(pVect2v) + dimension); + const float norm_v1 = load_unaligned(static_cast(pVect1v) + dimension); + const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } diff --git a/src/VecSim/spaces/L2/L2.cpp b/src/VecSim/spaces/L2/L2.cpp index 4fb67abd3..015f2200d 100644 --- a/src/VecSim/spaces/L2/L2.cpp +++ b/src/VecSim/spaces/L2/L2.cpp @@ -31,10 +31,10 @@ float SQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) // Get the raw inner product using the common implementation const float ip = SQ8_FP32_InnerProduct_Impl(pVect1v, pVect2v, dimension); - // Get precomputed sum of squares from storage blob (pVect1 is SQ8) + // Storage metadata follows a byte payload and is not necessarily float-aligned. const auto *pVect1 = static_cast(pVect1v); - const float *params = reinterpret_cast(pVect1 + dimension); - const float x_sum_sq = params[sq8::SUM_SQUARES]; + const float x_sum_sq = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Get precomputed sum of squares from query blob (pVect2 is FP32) const auto *pVect2 = static_cast(pVect2v); @@ -189,9 +189,9 @@ float SQ8_SQ8_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) // Get precomputed sum of squares from both vectors // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] const float sum_sq_1 = - *reinterpret_cast(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); const float sum_sq_2 = - *reinterpret_cast(pVect2 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVect2 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Use the common inner product implementation const float ip = SQ8_SQ8_InnerProduct_Impl(pVect1v, pVect2v, dimension); diff --git a/src/VecSim/spaces/L2/L2_AVX2_FMA_SQ8_FP32.h b/src/VecSim/spaces/L2/L2_AVX2_FMA_SQ8_FP32.h index 6e7a3ca59..0f9d5bde9 100644 --- a/src/VecSim/spaces/L2/L2_AVX2_FMA_SQ8_FP32.h +++ b/src/VecSim/spaces/L2/L2_AVX2_FMA_SQ8_FP32.h @@ -35,8 +35,8 @@ float SQ8_FP32_L2SqrSIMD16_AVX2_FMA(const void *pVect1v, const void *pVect2v, si // Get precomputed sum of squares from storage blob (pVect1v is SQ8 storage) const uint8_t *pVect1 = static_cast(pVect1v); - const float *params = reinterpret_cast(pVect1 + dimension); - const float x_sum_sq = params[sq8::SUM_SQUARES]; + const float x_sum_sq = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Get precomputed sum of squares from query blob (pVect2v is FP32 query) const float y_sum_sq = static_cast(pVect2v)[dimension + sq8::SUM_SQUARES_QUERY]; diff --git a/src/VecSim/spaces/L2/L2_AVX2_SQ8_FP32.h b/src/VecSim/spaces/L2/L2_AVX2_SQ8_FP32.h index 96eb0d02c..d08474f71 100644 --- a/src/VecSim/spaces/L2/L2_AVX2_SQ8_FP32.h +++ b/src/VecSim/spaces/L2/L2_AVX2_SQ8_FP32.h @@ -35,8 +35,8 @@ float SQ8_FP32_L2SqrSIMD16_AVX2(const void *pVect1v, const void *pVect2v, size_t // Get precomputed sum of squares from storage blob (pVect1v is SQ8 storage) const uint8_t *pVect1 = static_cast(pVect1v); - const float *params = reinterpret_cast(pVect1 + dimension); - const float x_sum_sq = params[sq8::SUM_SQUARES]; + const float x_sum_sq = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Get precomputed sum of squares from query blob (pVect2v is FP32 query) const float y_sum_sq = static_cast(pVect2v)[dimension + sq8::SUM_SQUARES_QUERY]; diff --git a/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_FP32.h b/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_FP32.h index 721789c8e..f1c86d689 100644 --- a/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_FP32.h +++ b/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_FP32.h @@ -35,8 +35,8 @@ float SQ8_FP32_L2SqrSIMD16_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *p // Get precomputed sum of squares from storage blob (pVect1v is SQ8 storage) const uint8_t *pVect1 = static_cast(pVect1v); - const float *params = reinterpret_cast(pVect1 + dimension); - const float x_sum_sq = params[sq8::SUM_SQUARES]; + const float x_sum_sq = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Get precomputed sum of squares from query blob (pVect2v is FP32 query) const float y_sum_sq = static_cast(pVect2v)[dimension + sq8::SUM_SQUARES_QUERY]; diff --git a/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_SQ8.h b/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_SQ8.h index df3043bf5..6a862b2ed 100644 --- a/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_SQ8.h +++ b/src/VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_SQ8.h @@ -34,8 +34,10 @@ float SQ8_SQ8_L2SqrSIMD64_AVX512F_BW_VL_VNNI(const void *pVec1v, const void *pVe const uint8_t *pVec2 = static_cast(pVec2v); // Get precomputed sum of squares from both vectors // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] - const float sum_sq_1 = *reinterpret_cast(pVec1 + dimension + 3 * sizeof(float)); - const float sum_sq_2 = *reinterpret_cast(pVec2 + dimension + 3 * sizeof(float)); + const float sum_sq_1 = + load_unaligned(pVec1 + dimension + sq8::SUM_SQUARES * sizeof(float)); + const float sum_sq_2 = + load_unaligned(pVec2 + dimension + sq8::SUM_SQUARES * sizeof(float)); // L2² = ||x||² + ||y||² - 2*IP(x, y) return sum_sq_1 + sum_sq_2 - 2.0f * ip; diff --git a/src/VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h b/src/VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h index 7de9f336a..6c55b3f03 100644 --- a/src/VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h +++ b/src/VecSim/spaces/L2/L2_NEON_DOTPROD_SQ8_SQ8.h @@ -38,9 +38,9 @@ float SQ8_SQ8_L2SqrSIMD64_NEON_DOTPROD(const void *pVec1v, const void *pVec2v, s // Get precomputed sum of squares from both vectors // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] const float sum_sq_1 = - *reinterpret_cast(pVec1 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVec1 + dimension + sq8::SUM_SQUARES * sizeof(float)); const float sum_sq_2 = - *reinterpret_cast(pVec2 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVec2 + dimension + sq8::SUM_SQUARES * sizeof(float)); // L2² = ||x||² + ||y||² - 2*IP(x, y) return sum_sq_1 + sum_sq_2 - 2.0f * ip; diff --git a/src/VecSim/spaces/L2/L2_NEON_SQ8_FP32.h b/src/VecSim/spaces/L2/L2_NEON_SQ8_FP32.h index e98beb13e..f6f7a6bc0 100644 --- a/src/VecSim/spaces/L2/L2_NEON_SQ8_FP32.h +++ b/src/VecSim/spaces/L2/L2_NEON_SQ8_FP32.h @@ -36,8 +36,8 @@ float SQ8_FP32_L2SqrSIMD16_NEON(const void *pVect1v, const void *pVect2v, size_t // Get precomputed sum of squares from storage blob (pVect1v is SQ8 storage) const uint8_t *pVect1 = static_cast(pVect1v); - const float *params = reinterpret_cast(pVect1 + dimension); - const float x_sum_sq = params[sq8::SUM_SQUARES]; + const float x_sum_sq = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Get precomputed sum of squares from query blob (pVect2v is FP32 query) const float y_sum_sq = static_cast(pVect2v)[dimension + sq8::SUM_SQUARES_QUERY]; diff --git a/src/VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h b/src/VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h index e86838404..2750997f7 100644 --- a/src/VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h +++ b/src/VecSim/spaces/L2/L2_NEON_SQ8_SQ8.h @@ -37,9 +37,9 @@ float SQ8_SQ8_L2SqrSIMD64_NEON(const void *pVec1v, const void *pVec2v, size_t di // Get precomputed sum of squares from both vectors // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] const float sum_sq_1 = - *reinterpret_cast(pVec1 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVec1 + dimension + sq8::SUM_SQUARES * sizeof(float)); const float sum_sq_2 = - *reinterpret_cast(pVec2 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVec2 + dimension + sq8::SUM_SQUARES * sizeof(float)); // L2² = ||x||² + ||y||² - 2*IP(x, y) return sum_sq_1 + sum_sq_2 - 2.0f * ip; diff --git a/src/VecSim/spaces/L2/L2_SSE4_SQ8_FP32.h b/src/VecSim/spaces/L2/L2_SSE4_SQ8_FP32.h index 29c662786..3a3a4d12d 100644 --- a/src/VecSim/spaces/L2/L2_SSE4_SQ8_FP32.h +++ b/src/VecSim/spaces/L2/L2_SSE4_SQ8_FP32.h @@ -35,8 +35,8 @@ float SQ8_FP32_L2SqrSIMD16_SSE4(const void *pVect1v, const void *pVect2v, size_t // Get precomputed sum of squares from storage blob (pVect1v is SQ8 storage) const uint8_t *pVect1 = static_cast(pVect1v); - const float *params = reinterpret_cast(pVect1 + dimension); - const float x_sum_sq = params[sq8::SUM_SQUARES]; + const float x_sum_sq = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Get precomputed sum of squares from query blob (pVect2v is FP32 query) const float y_sum_sq = static_cast(pVect2v)[dimension + sq8::SUM_SQUARES_QUERY]; diff --git a/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h b/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h index 0ae9fec74..3f95c4ae4 100644 --- a/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h +++ b/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h @@ -37,8 +37,8 @@ float SQ8_FP32_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t di // Get precomputed sum of squares from storage blob (pVect1v is SQ8 storage) const uint8_t *pVect1 = static_cast(pVect1v); - const float *params = reinterpret_cast(pVect1 + dimension); - const float x_sum_sq = params[sq8::SUM_SQUARES]; + const float x_sum_sq = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); // Get precomputed sum of squares from query blob (pVect2v is FP32 query) const float y_sum_sq = static_cast(pVect2v)[dimension + sq8::SUM_SQUARES_QUERY]; diff --git a/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h b/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h index 90801f82a..5fe194d80 100644 --- a/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h +++ b/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h @@ -38,9 +38,9 @@ float SQ8_SQ8_L2SqrSIMD_SVE(const void *pVec1v, const void *pVec2v, size_t dimen // Get precomputed sum of squares from both vectors // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] const float sum_sq_1 = - *reinterpret_cast(pVec1 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVec1 + dimension + sq8::SUM_SQUARES * sizeof(float)); const float sum_sq_2 = - *reinterpret_cast(pVec2 + dimension + sq8::SUM_SQUARES * sizeof(float)); + load_unaligned(pVec2 + dimension + sq8::SUM_SQUARES * sizeof(float)); // L2² = ||x||² + ||y||² - 2*IP(x, y) return sum_sq_1 + sum_sq_2 - 2.0f * ip; diff --git a/src/VecSim/spaces/normalize/normalize_naive.h b/src/VecSim/spaces/normalize/normalize_naive.h index 85bdc88c1..01269a3eb 100644 --- a/src/VecSim/spaces/normalize/normalize_naive.h +++ b/src/VecSim/spaces/normalize/normalize_naive.h @@ -12,6 +12,7 @@ #include "VecSim/types/float16.h" #include "compute_norm.h" #include +#include #include using bfloat16 = vecsim_types::bfloat16; @@ -82,8 +83,8 @@ static inline void integer_normalizeVector(void *vec, const size_t dim) { float norm = IntegralType_ComputeNorm(input_vector, dim); - // Store norm at the end of the vector. - *reinterpret_cast(input_vector + dim) = norm; + // The norm follows a one-byte element payload and may be unaligned for odd dimensions. + std::memcpy(input_vector + dim, &norm, sizeof(norm)); } } // namespace spaces diff --git a/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp32.cpp b/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp32.cpp index 320818c02..9565c0417 100644 --- a/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp32.cpp +++ b/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp32.cpp @@ -15,8 +15,8 @@ class BM_VecSimSpaces_SQ8_FP32 : public benchmark::Fixture { protected: std::mt19937 rng; size_t dim; - float *v1; - uint8_t *v2; + uint8_t *v1; + float *v2; public: BM_VecSimSpaces_SQ8_FP32() { rng.seed(47); } @@ -24,17 +24,17 @@ class BM_VecSimSpaces_SQ8_FP32 : public benchmark::Fixture { void SetUp(const ::benchmark::State &state) { dim = state.range(0); - size_t query_size = dim + sq8::query_metadata_count(); - v1 = new float[query_size]; - test_utils::populate_sq8_fp32_query(v1, dim, true, 123); size_t quantized_size = dim * sizeof(uint8_t) + sq8::storage_metadata_count() * sizeof(float); - v2 = new uint8_t[quantized_size]; - test_utils::populate_float_vec_to_sq8_with_metadata(v2, dim, true, 1234); + v1 = new uint8_t[quantized_size]; + test_utils::populate_float_vec_to_sq8_with_metadata(v1, dim, true, 1234); + size_t query_size = dim + sq8::query_metadata_count(); + v2 = new float[query_size]; + test_utils::populate_sq8_fp32_query(v2, dim, true, 123); } void TearDown(const ::benchmark::State &state) { - delete v1; - delete v2; + delete[] v1; + delete[] v2; } }; diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index eb465fa92..03fc4b91f 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -1149,19 +1149,22 @@ TEST(PreprocessorsTest, QuantizationTestAllEntriesEqual) { } // Verify metadata: min_val = 3.5f, delta = 1.0f (fallback when diff == 0) - const float *metadata = reinterpret_cast(quantized + dim); - ASSERT_FLOAT_EQ(metadata[sq8::MIN_VAL], 3.5f); // min_val - ASSERT_FLOAT_EQ(metadata[sq8::DELTA], 1.0f); // delta (fallback) + const auto *metadata = quantized + dim; + const float min_val = load_unaligned(metadata + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(metadata + sq8::DELTA * sizeof(float)); + ASSERT_FLOAT_EQ(min_val, 3.5f); + ASSERT_FLOAT_EQ(delta, 1.0f); // Verify sum and sum_squares for L2 metric float expected_sum = 3.5f * dim; float expected_sum_squares = 3.5f * 3.5f * dim; - ASSERT_FLOAT_EQ(metadata[sq8::SUM], expected_sum); // sum - ASSERT_FLOAT_EQ(metadata[sq8::SUM_SQUARES], expected_sum_squares); // sum_squares + ASSERT_FLOAT_EQ(load_unaligned(metadata + sq8::SUM * sizeof(float)), expected_sum); + ASSERT_FLOAT_EQ(load_unaligned(metadata + sq8::SUM_SQUARES * sizeof(float)), + expected_sum_squares); // Reconstruct and verify: min + quantized * delta = 3.5 + 0 * 1 = 3.5 for (size_t i = 0; i < dim; ++i) { - float reconstructed = metadata[sq8::MIN_VAL] + quantized[i] * metadata[sq8::DELTA]; + float reconstructed = min_val + quantized[i] * delta; ASSERT_FLOAT_EQ(reconstructed, original_blob[i]); } diff --git a/tests/unit/test_index_test_utils.cpp b/tests/unit/test_index_test_utils.cpp index 1134245b8..c2ffed687 100644 --- a/tests/unit/test_index_test_utils.cpp +++ b/tests/unit/test_index_test_utils.cpp @@ -116,8 +116,8 @@ class Int8IndexTestUtilsTest : public IndexTestUtilsTest { // compute expected norm using the original vector float expected_norm = test_utils::integral_compute_norm(vectors[i * vec_per_label + j].data(), dim); - const float *stored_norm = reinterpret_cast(stored_vec + dim); - ASSERT_EQ(*stored_norm, expected_norm) << "wrong vector norm for vector id:" << j; + const float stored_norm = load_unaligned(stored_vec + dim); + ASSERT_EQ(stored_norm, expected_norm) << "wrong vector norm for vector id:" << j; } } } diff --git a/tests/unit/test_int8.cpp b/tests/unit/test_int8.cpp index 47a700489..71c93bf4d 100644 --- a/tests/unit/test_int8.cpp +++ b/tests/unit/test_int8.cpp @@ -422,7 +422,7 @@ void INT8Test::metrics_test(params_t index_params) { if (metric == VecSimMetric_Cosine) { // compare with the norm stored in the index vector const int8_t *index_vector = static_cast(this->GetDataByInternalId(i)); - float index_vector_norm = *(reinterpret_cast(index_vector + dim)); + float index_vector_norm = load_unaligned(index_vector + dim); float vector_norm = spaces::IntegralType_ComputeNorm(vector, dim); ASSERT_EQ(index_vector_norm, vector_norm) << "wrong vector norm for vector id:" << i; } else if (metric == VecSimMetric_IP) { @@ -1050,7 +1050,7 @@ TEST_F(INT8TieredTest, CosineBlobCorrectness) { float vector_norm = spaces::IntegralType_ComputeNorm(vector, dim); auto verify_norm = [&](const int8_t *input_vector, float expected_norm) { - float vectors_stored_norm = *(reinterpret_cast(input_vector + dim)); + float vectors_stored_norm = load_unaligned(input_vector + dim); ASSERT_EQ(vectors_stored_norm, expected_norm) << "wrong vector norm"; }; diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 8c4681ae7..8e53c83d1 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -292,8 +292,10 @@ TEST_F(SpacesTest, int8_Cosine_no_optimization_func_test) { test_utils::populate_int8_vec(v2, dim, 123); // write the norm at the end of the vector - *(float *)(v1 + dim) = test_utils::integral_compute_norm(v1, dim); - *(float *)(v2 + dim) = test_utils::integral_compute_norm(v2, dim); + const float norm_v1 = test_utils::integral_compute_norm(v1, dim); + const float norm_v2 = test_utils::integral_compute_norm(v2, dim); + std::memcpy(v1 + dim, &norm_v1, sizeof(norm_v1)); + std::memcpy(v2 + dim, &norm_v2, sizeof(norm_v2)); float dist = INT8_Cosine((const void *)v1, (const void *)v2, dim); ASSERT_NEAR(dist, 0.0, 0.000001); @@ -309,8 +311,10 @@ TEST_F(SpacesTest, uint8_Cosine_no_optimization_func_test) { test_utils::populate_uint8_vec(v2, dim, 123); // write the norm at the end of the vector - *(float *)(v1 + dim) = test_utils::integral_compute_norm(v1, dim); - *(float *)(v2 + dim) = test_utils::integral_compute_norm(v2, dim); + const float norm_v1 = test_utils::integral_compute_norm(v1, dim); + const float norm_v2 = test_utils::integral_compute_norm(v2, dim); + std::memcpy(v1 + dim, &norm_v1, sizeof(norm_v1)); + std::memcpy(v2 + dim, &norm_v2, sizeof(norm_v2)); float dist = UINT8_Cosine((const void *)v1, (const void *)v2, dim); ASSERT_NEAR(dist, 0.0, 0.000001); @@ -367,6 +371,42 @@ TEST_F(SpacesTest, SQ8_FP32_l2sqr_no_optimization_func_test) { ASSERT_NEAR(dist, baseline, 0.01) << "SQ8_FP32_L2Sqr failed to match expected distance"; } +TEST_F(SpacesTest, SQ8_FP32_odd_dim_unaligned_metadata_test) { + for (const size_t dim : {1UL, 5UL, 7UL, 15UL}) { + const size_t query_size = dim + sq8::query_metadata_count(); + std::vector query(query_size); + test_utils::populate_sq8_fp32_query(query.data(), dim, false, 1234); + + const size_t storage_size = + dim + sq8::storage_metadata_count() * sizeof(float); + std::vector allocation(storage_size + alignof(float)); + auto *storage = allocation.data(); + while (reinterpret_cast(storage) % alignof(float) != 0) { + ++storage; + } + test_utils::populate_float_vec_to_sq8_with_metadata(storage, dim, false, 5678); + + const auto *metadata = storage + dim; + ASSERT_NE(reinterpret_cast(metadata) % alignof(float), 0u); + + const float expected_ip = + test_utils::SQ8_FP32_NotOptimized_InnerProduct(storage, query.data(), dim); + const float expected_l2 = + test_utils::SQ8_FP32_NotOptimized_L2Sqr(storage, query.data(), dim); + + EXPECT_NEAR(SQ8_FP32_InnerProduct(storage, query.data(), dim), expected_ip, 0.01) + << "scalar IP with dim " << dim; + EXPECT_NEAR(SQ8_FP32_L2Sqr(storage, query.data(), dim), expected_l2, 0.01) + << "scalar L2 with dim " << dim; + EXPECT_NEAR(IP_SQ8_FP32_GetDistFunc(dim, nullptr)(storage, query.data(), dim), expected_ip, + 0.01) + << "dispatched IP with dim " << dim; + EXPECT_NEAR(L2_SQ8_FP32_GetDistFunc(dim, nullptr)(storage, query.data(), dim), expected_l2, + 0.01) + << "dispatched L2 with dim " << dim; + } +} + /* ======================== Tests SQ8-FP16 ========================= */ TEST_F(SpacesTest, SQ8_FP16_ip_no_optimization_norm_func_test) { @@ -1730,8 +1770,10 @@ TEST_P(INT8SpacesOptimizationTest, INT8CosineTest) { test_utils::populate_int8_vec(v2, dim, 1234); // write the norm at the end of the vector - *(float *)(v1 + dim) = test_utils::integral_compute_norm(v1, dim); - *(float *)(v2 + dim) = test_utils::integral_compute_norm(v2, dim); + const float norm_v1 = test_utils::integral_compute_norm(v1, dim); + const float norm_v2 = test_utils::integral_compute_norm(v2, dim); + std::memcpy(v1 + dim, &norm_v1, sizeof(norm_v1)); + std::memcpy(v2 + dim, &norm_v2, sizeof(norm_v2)); dist_func_t arch_opt_func; float baseline = INT8_Cosine(v1, v2, dim); @@ -1985,8 +2027,10 @@ TEST_P(UINT8SpacesOptimizationTest, UINT8CosineTest) { test_utils::populate_uint8_vec(v2, dim, 1234); // write the norm at the end of the vector - *(float *)(v1 + dim) = test_utils::integral_compute_norm(v1, dim); - *(float *)(v2 + dim) = test_utils::integral_compute_norm(v2, dim); + const float norm_v1 = test_utils::integral_compute_norm(v1, dim); + const float norm_v2 = test_utils::integral_compute_norm(v2, dim); + std::memcpy(v1 + dim, &norm_v1, sizeof(norm_v1)); + std::memcpy(v2 + dim, &norm_v2, sizeof(norm_v2)); dist_func_t arch_opt_func; float baseline = UINT8_Cosine(v1, v2, dim); @@ -2079,8 +2123,10 @@ TEST_P(UINT8SpacesOptimizationTest, UINT8_full_range_test) { } // write the norm at the end of the vector - *(float *)(v1 + dim) = test_utils::integral_compute_norm(v1, dim); - *(float *)(v2 + dim) = test_utils::integral_compute_norm(v2, dim); + const float norm_v1 = test_utils::integral_compute_norm(v1, dim); + const float norm_v2 = test_utils::integral_compute_norm(v2, dim); + std::memcpy(v1 + dim, &norm_v1, sizeof(norm_v1)); + std::memcpy(v2 + dim, &norm_v2, sizeof(norm_v2)); float baseline_l2 = UINT8_L2Sqr(v1, v2, dim); float baseline_ip = UINT8_InnerProduct(v1, v2, dim); diff --git a/tests/unit/test_uint8.cpp b/tests/unit/test_uint8.cpp index 7515d6b8b..e51ff9a93 100644 --- a/tests/unit/test_uint8.cpp +++ b/tests/unit/test_uint8.cpp @@ -422,7 +422,7 @@ void UINT8Test::metrics_test(params_t index_params) { if (metric == VecSimMetric_Cosine) { // compare with the norm stored in the index vector auto *index_vector = static_cast(this->GetDataByInternalId(i)); - float index_vector_norm = *(reinterpret_cast(index_vector + dim)); + float index_vector_norm = load_unaligned(index_vector + dim); float vector_norm = spaces::IntegralType_ComputeNorm(vector, dim); ASSERT_FLOAT_EQ(index_vector_norm, vector_norm) << "wrong norm for vector id:" << i; } else if (metric == VecSimMetric_IP) { diff --git a/tests/utils/tests_utils.h b/tests/utils/tests_utils.h index 2650dd4d2..67d93177c 100644 --- a/tests/utils/tests_utils.h +++ b/tests/utils/tests_utils.h @@ -86,8 +86,8 @@ static float SQ8_FP32_NotOptimized_InnerProduct(const void *pVect1v, const void const auto *pVect2 = static_cast(pVect2v); // FP32 query // Get quantization parameters from pVect1 (SQ8 storage) - const float min_val = *reinterpret_cast(pVect1 + dimension); - const float delta = *reinterpret_cast(pVect1 + dimension + sizeof(float)); + const float min_val = load_unaligned(pVect1 + dimension + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(pVect1 + dimension + sq8::DELTA * sizeof(float)); // Compute inner product with dequantization float res = 0.0f; for (size_t i = 0; i < dimension; i++) { @@ -119,12 +119,12 @@ static float SQ8_SQ8_NotOptimized_InnerProduct(const void *pVect1v, const void * const auto *pVect2 = static_cast(pVect2v); // Get quantization parameters from pVect1 - const float min_val1 = *reinterpret_cast(pVect1 + dimension); - const float delta1 = *reinterpret_cast(pVect1 + dimension + sizeof(float)); + const float min_val1 = load_unaligned(pVect1 + dimension + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(pVect1 + dimension + sq8::DELTA * sizeof(float)); // Get quantization parameters from pVect2 - const float min_val2 = *reinterpret_cast(pVect2 + dimension); - const float delta2 = *reinterpret_cast(pVect2 + dimension + sizeof(float)); + const float min_val2 = load_unaligned(pVect2 + dimension + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(pVect2 + dimension + sq8::DELTA * sizeof(float)); // Compute inner product with dequantization float res = 0.0f; @@ -152,10 +152,10 @@ static float SQ8_SQ8_NotOptimized_L2Sqr(const void *pVect1v, const void *pVect2v // Extract metadata from the end of vectors // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] - const float min1 = *reinterpret_cast(pVect1 + dimension); - const float delta1 = *reinterpret_cast(pVect1 + dimension + sizeof(float)); - const float min2 = *reinterpret_cast(pVect2 + dimension); - const float delta2 = *reinterpret_cast(pVect2 + dimension + sizeof(float)); + const float min1 = load_unaligned(pVect1 + dimension + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(pVect1 + dimension + sq8::DELTA * sizeof(float)); + const float min2 = load_unaligned(pVect2 + dimension + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(pVect2 + dimension + sq8::DELTA * sizeof(float)); // Compute L2 distance with dequantization float res = 0.0f; @@ -201,11 +201,11 @@ static void quantize_float_vec_to_sq8_with_metadata(const float *v, size_t dim, } // Store parameters: [min, delta, sum, square_sum] - float *params = reinterpret_cast(qv + dim); - params[sq8::MIN_VAL] = min_val; - params[sq8::DELTA] = delta; - params[sq8::SUM] = sum; - params[sq8::SUM_SQUARES] = square_sum; + auto *params = qv + dim; + std::memcpy(params + sq8::MIN_VAL * sizeof(float), &min_val, sizeof(float)); + std::memcpy(params + sq8::DELTA * sizeof(float), &delta, sizeof(float)); + std::memcpy(params + sq8::SUM * sizeof(float), &sum, sizeof(float)); + std::memcpy(params + sq8::SUM_SQUARES * sizeof(float), &square_sum, sizeof(float)); } // Preprocess fp32 query for SQ8 IP/Cosine/L2 space. @@ -246,8 +246,8 @@ static float SQ8_FP32_NotOptimized_L2Sqr(const void *pVect1v, const void *pVect2 const auto *pVect2 = static_cast(pVect2v); // FP32 query // Get quantization parameters from pVect1 (SQ8 storage) - const float min_val = *reinterpret_cast(pVect1 + dimension); - const float delta = *reinterpret_cast(pVect1 + dimension + sizeof(float)); + const float min_val = load_unaligned(pVect1 + dimension + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(pVect1 + dimension + sq8::DELTA * sizeof(float)); // Compute L2 squared with dequantization float res = 0.0f;