From 0e8427a22ab0aecb3f7efd85432fff4a46f833a0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 4 Nov 2026 21:10:22 -0800 Subject: [PATCH 1/5] CPU: ik_llama.cpp GEMM kernels for k-quant / i-quant weights (ggml-patches/0018) Ports the classic iqk GEMM kernels of ik_llama.cpp (commit 009dce19, MIT, Iwan Kawrakow) into ggml-cpu as src/ggml-cpu/iqk/: Q2_K, Q3_K, Q4_K, Q5_K, Q6_K, IQ4_XS, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ1_S and IQ1_M weights times Q8_K activations, for AVX2, AVX512 (F/BW/DQ/VL/VNNI) and NEON with dotprod. They read the upstream block layouts and the Q8_K rows ggml already makes, form the same integer products as vec_dot and only accumulate the floats in another order; ik's re-quantizing Q8_K_R8 conversion path is not ported. Several activation rows share each weight block decode (up to 8), instead of one row by one column per vec_dot call. Hooked in ggml_compute_forward_mul_mat after src1 is converted to Q8_K, in place of the vec_dot loop. On x86 the tiled K-quant path keeps taking ne11 >= 8 first; on ARM, which has no tiled path, these kernels take every ne11. Variants without AVX2 (or NEON dotprod) and MSVC builds compile a stub and keep vec_dot. Repacked weights do not reach this code. GGML_CPU_IQK=0 disables, GGML_CPU_IQK=2 runs them ahead of the tiled path, GGML_CPU_IQK_TYPES=q2_K,iq2_xs,... restricts the types, GGML_CPU_IQK_DEBUG=1 logs the first use per type. --- .../ggml-patches/0018-ggml-cpu-iqk-gemm.patch | 3855 +++++++++++++++++ 1 file changed, 3855 insertions(+) create mode 100644 scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch diff --git a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch new file mode 100644 index 000000000..fa138d843 --- /dev/null +++ b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch @@ -0,0 +1,3855 @@ +diff --git a/src/ggml-cpu/CMakeLists.txt b/src/ggml-cpu/CMakeLists.txt +index d347757e..9bcd6009 100644 +--- a/src/ggml-cpu/CMakeLists.txt ++++ b/src/ggml-cpu/CMakeLists.txt +@@ -45,6 +45,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) + ggml-cpu/tiled/tiled.h + ggml-cpu/tiled/tiled-kernel.cpp + ggml-cpu/tiled/tiled-kernel.h ++ ggml-cpu/iqk/iqk_gemm.cpp ++ ggml-cpu/iqk/iqk_gemm.h ++ ggml-cpu/iqk/iqk_common.h ++ ggml-cpu/iqk/iqk_gemm_kquants.cpp ++ ggml-cpu/iqk/iqk_gemm_iquants.cpp ++ ggml-cpu/iqk/iqk_gemm_1bit.cpp + ggml-cpu/ggml-cpu-impl.h + ggml-cpu/common.h + ggml-cpu/binary-ops.h +diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c +index c10f0679..446a361e 100644 +--- a/src/ggml-cpu/ggml-cpu.c ++++ b/src/ggml-cpu/ggml-cpu.c +@@ -15,6 +15,7 @@ + #include "ggml.h" + #include "common.h" + #include "tiled/tiled.h" ++#include "iqk/iqk_gemm.h" + + #if defined(_MSC_VER) || defined(__MINGW32__) + #include // using malloc.h with MSC/MINGW +@@ -1455,8 +1456,14 @@ void ggml_compute_forward_mul_mat( + return; + } + ++ // ggml_cpu_iqk_mul_mat (ik_llama.cpp kernels) runs below for its weight types once src1 is in Q8_K, in place of ++ // vec_dot. The x86 tiled K-quant path still goes first (it takes ne11 >= 8) unless GGML_CPU_IQK=2. ++ const bool iqk = !params->use_ref && ggml_cpu_iqk_type_enabled(src0->type) && ++ type_traits_cpu[src0->type].vec_dot_type == GGML_TYPE_Q8_K && dst->type == GGML_TYPE_F32 && ++ src0->ne[0] % 256 == 0; ++ + // If tiled is supported, it will execute the full op here and we return +- if (ggml_compute_forward_mul_mat_tiled(params, dst)) { ++ if (!(iqk && ggml_cpu_iqk_before_tiled()) && ggml_compute_forward_mul_mat_tiled(params, dst)) { + return; + } + +@@ -1597,6 +1604,26 @@ UseGgmlGemm1:; + UseGgmlGemm2:; + #endif + ++ if (iqk) { ++ // same row / broadcast walk as the llamafile hook above; src1 rows are Q8_K now ++ const int64_t r2i = ne12 / ne02; ++ const int64_t r3i = ne13 / ne03; ++ const bool in_place = src1->type == vec_dot_type; ++ const size_t row_size = ggml_row_size(vec_dot_type, ne10); ++ for (int64_t i13 = 0; i13 < ne13; i13++) { ++ for (int64_t i12 = 0; i12 < ne12; i12++) { ++ const char * b = in_place ? (const char *) src1->data + i12*nb12 + i13*nb13 ++ : (const char *) params->wdata + (i12*ne11 + i13*ne12*ne11)*row_size; ++ ggml_cpu_iqk_mul_mat(src0->type, ne01, ne11, ne00, ++ (const char *) src0->data + i12/r2i*nb02 + i13/r3i*nb03, nb01, ++ b, in_place ? nb11 : row_size, ++ (float *) ((char *) dst->data + i12*nb2 + i13*nb3), nb1/sizeof(float), ++ ith, nth); ++ } ++ } ++ return; ++ } ++ + // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) + const int64_t nr0 = ne0; + +@@ -4126,6 +4153,7 @@ void ggml_cpu_init(void) { + #ifdef GGML_USE_LLAMAFILE + ggml_cpu_sgemm_types_init(); + #endif ++ ggml_cpu_iqk_init(); + + GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0); + +diff --git a/src/ggml-cpu/ggml-cpu.cpp b/src/ggml-cpu/ggml-cpu.cpp +index 9cef95e1..31de8537 100644 +--- a/src/ggml-cpu/ggml-cpu.cpp ++++ b/src/ggml-cpu/ggml-cpu.cpp +@@ -5,6 +5,7 @@ + #include "traits.h" + #include "ggml-impl.h" + #include "amx/amx.h" ++#include "iqk/iqk_gemm.h" + + #include + #include +@@ -708,6 +709,13 @@ static void * ggml_backend_cpu_get_proc_address(ggml_backend_reg_t reg, const ch + if (strcmp(name, "ggml_backend_cpu_set_use_ref") == 0) { + return (void *)ggml_backend_cpu_set_use_ref; + } ++ // iqk GEMM test hooks: int64_t (*)(int type) engagement count, void (*)(uint64_t mask) enabled types ++ if (strcmp(name, "ggml_backend_cpu_iqk_count") == 0) { ++ return (void *)ggml_cpu_iqk_count; ++ } ++ if (strcmp(name, "ggml_backend_cpu_iqk_set_types") == 0) { ++ return (void *)ggml_cpu_iqk_set_types; ++ } + + // threadpool - TODO: move to ggml-base + if (strcmp(name, "ggml_threadpool_new") == 0) { +diff --git a/src/ggml-cpu/iqk/iqk_common.h b/src/ggml-cpu/iqk/iqk_common.h +new file mode 100644 +index 00000000..478cf1a4 +--- /dev/null ++++ b/src/ggml-cpu/iqk/iqk_common.h +@@ -0,0 +1,754 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++// Adapted from ik_llama.cpp (https://github.com/ikawrakow/ik_llama.cpp) commit 009dce19, ++// ggml/src/iqk/{iqk_config.h, iqk_common.h, iqk_gemm_kquants.cpp, iqk_gemm_iquants.cpp, iqk_gemm_1bit.cpp, iqk_mul_mat.cpp}. ++// Only the kernels that read the upstream ggml block layouts with upstream Q8_K activations are kept. ++// ++// Permission is hereby granted, free of charge, to any person obtaining a copy ++// of this software and associated documentation files (the "Software"), to deal ++// in the Software without restriction, including without limitation the rights ++// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell ++// copies of the Software, and to permit persons to whom the Software is ++// furnished to do so, subject to the following conditions: ++// ++// The above copyright notice and this permission notice shall be included in all ++// copies or substantial portions of the Software. ++// ++// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ++// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++// SOFTWARE. ++// ++// Shared helpers of the iqk GEMM kernels (iqk_config.h + iqk_common.h, trimmed to the classic Q8_K kernels). ++ ++#pragma once ++ ++#if defined IQK_IMPLEMENT ++#undef IQK_IMPLEMENT ++#endif ++ ++// MSVC is left on the vec_dot path: these kernels are only built and checked with GCC / Clang. ++#if (defined __AVX2__ || (defined __aarch64__ && defined __ARM_FEATURE_DOTPROD)) && !(defined(_MSC_VER) && !defined(__clang__)) ++#define IQK_IMPLEMENT ++#endif ++ ++#ifdef IQK_IMPLEMENT ++ ++#define IQK_NOINLINE __attribute__((__noinline__)) ++#define IQK_ALWAYS_INLINE __attribute__((__always_inline__)) ++ ++#if defined __x86_64__ ++#if defined HAVE_FANCY_SIMD ++ #undef HAVE_FANCY_SIMD ++#endif ++#if defined(__AVX512F__) && defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512BW__) && defined(__AVX512DQ__) ++ #define HAVE_FANCY_SIMD ++#endif ++#endif ++ ++#include ++#include ++#include ++#include ++ ++#define GGML_COMMON_DECL_CPP ++#define GGML_COMMON_IMPL_CPP ++#include "ggml-common.h" ++#include "ggml-impl.h" ++#include "ggml-cpu-impl.h" ++#include "simd-mappings.h" ++ ++// the kernels convert the fp16 block scales with the CPU backend's fast conversion ++#undef GGML_FP16_TO_FP32 ++#define GGML_FP16_TO_FP32(x) GGML_CPU_FP16_TO_FP32(x) ++ ++struct DataInfo { ++ float * s; ++ const char * cy; ++ size_t bs; ++ size_t by; ++ int cur_y = 0; ++ ++ inline const char * src1_row(int iy) const { ++ return cy + (cur_y + iy)*by; ++ } ++ ++ inline void store(int ix, int iy, float result) const { ++ *(dst_row(iy) + ix) = result; ++ } ++#ifdef __AVX__ ++ inline void store(int ix, int iy, __m128 result) const { ++ _mm_storeu_ps(dst_row(iy) + ix, result); ++ } ++ inline void store(int ix, int iy, __m256 result) const { ++ _mm256_storeu_ps(dst_row(iy) + ix, result); ++ } ++#endif ++#ifdef __AVX512F__ ++ inline void store(int ix, int iy, __m512 result) const { ++ _mm512_storeu_ps(dst_row(iy) + ix, result); ++ } ++#endif ++#ifdef __ARM_NEON ++ inline void store(int ix, int iy, float32x4_t result) const { ++ vst1q_f32(dst_row(iy) + ix, result); ++ } ++#endif ++ inline float * dst_row(int iy) const { ++ return s + (cur_y + iy)*bs; ++ } ++}; ++ ++typedef void (*mul_mat_t)(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x); ++ ++#define IQK_MAX_NY 8 ++ ++#define IQK_SET_MUL_MAT_FUNCTIONS_T(kernel, Dequantizer, funcs) \ ++ funcs[0] = kernel;\ ++ funcs[1] = kernel;\ ++ funcs[2] = kernel;\ ++ funcs[3] = kernel;\ ++ funcs[4] = kernel;\ ++ funcs[5] = kernel;\ ++ funcs[6] = kernel;\ ++ funcs[7] = kernel;\ ++ ++#define IQK_SET_MUL_MAT_FUNCTIONS(kernel, funcs) \ ++ funcs[0] = kernel<1>;\ ++ funcs[1] = kernel<2>;\ ++ funcs[2] = kernel<3>;\ ++ funcs[3] = kernel<4>;\ ++ funcs[4] = kernel<5>;\ ++ funcs[5] = kernel<6>;\ ++ funcs[6] = kernel<7>;\ ++ funcs[7] = kernel<8>;\ ++ ++// Each kernel family fills funcs[0..7] (kernels for 1..8 activation rows) for the types it handles. ++bool iqk_set_kernels_kquants(int typeA, mul_mat_t * funcs); ++bool iqk_set_kernels_iquants(int typeA, mul_mat_t * funcs); ++bool iqk_set_kernels_1bit(int typeA, mul_mat_t * funcs); ++ ++// ================================================================================================== ++ ++static inline void make_q4_scales(const uint8_t * scales8, uint32_t * aux32) { ++ const uint16_t * scales = (const uint16_t *)scales8; ++ const uint32_t a0 = scales[0] | (scales[1] << 16); ++ const uint32_t a1 = scales[2] | (scales[3] << 16); ++ const uint32_t a2 = scales[4] | (scales[5] << 16); ++ aux32[3] = ((a2 >> 4) & 0x0f0f0f0f) | ((a1 >> 2) & 0x30303030); ++ aux32[1] = ((a2 >> 0) & 0x0f0f0f0f) | ((a0 >> 2) & 0x30303030); ++ aux32[2] = a1 & 0x3f3f3f3f; ++ aux32[0] = a0 & 0x3f3f3f3f; ++} ++ ++const uint64_t keven_signs[128] = { ++ 0x0101010101010101, 0xff010101010101ff, 0xff0101010101ff01, 0x010101010101ffff, ++ 0xff01010101ff0101, 0x0101010101ff01ff, 0x0101010101ffff01, 0xff01010101ffffff, ++ 0xff010101ff010101, 0x01010101ff0101ff, 0x01010101ff01ff01, 0xff010101ff01ffff, ++ 0x01010101ffff0101, 0xff010101ffff01ff, 0xff010101ffffff01, 0x01010101ffffffff, ++ 0xff0101ff01010101, 0x010101ff010101ff, 0x010101ff0101ff01, 0xff0101ff0101ffff, ++ 0x010101ff01ff0101, 0xff0101ff01ff01ff, 0xff0101ff01ffff01, 0x010101ff01ffffff, ++ 0x010101ffff010101, 0xff0101ffff0101ff, 0xff0101ffff01ff01, 0x010101ffff01ffff, ++ 0xff0101ffffff0101, 0x010101ffffff01ff, 0x010101ffffffff01, 0xff0101ffffffffff, ++ 0xff01ff0101010101, 0x0101ff01010101ff, 0x0101ff010101ff01, 0xff01ff010101ffff, ++ 0x0101ff0101ff0101, 0xff01ff0101ff01ff, 0xff01ff0101ffff01, 0x0101ff0101ffffff, ++ 0x0101ff01ff010101, 0xff01ff01ff0101ff, 0xff01ff01ff01ff01, 0x0101ff01ff01ffff, ++ 0xff01ff01ffff0101, 0x0101ff01ffff01ff, 0x0101ff01ffffff01, 0xff01ff01ffffffff, ++ 0x0101ffff01010101, 0xff01ffff010101ff, 0xff01ffff0101ff01, 0x0101ffff0101ffff, ++ 0xff01ffff01ff0101, 0x0101ffff01ff01ff, 0x0101ffff01ffff01, 0xff01ffff01ffffff, ++ 0xff01ffffff010101, 0x0101ffffff0101ff, 0x0101ffffff01ff01, 0xff01ffffff01ffff, ++ 0x0101ffffffff0101, 0xff01ffffffff01ff, 0xff01ffffffffff01, 0x0101ffffffffffff, ++ 0xffff010101010101, 0x01ff0101010101ff, 0x01ff01010101ff01, 0xffff01010101ffff, ++ 0x01ff010101ff0101, 0xffff010101ff01ff, 0xffff010101ffff01, 0x01ff010101ffffff, ++ 0x01ff0101ff010101, 0xffff0101ff0101ff, 0xffff0101ff01ff01, 0x01ff0101ff01ffff, ++ 0xffff0101ffff0101, 0x01ff0101ffff01ff, 0x01ff0101ffffff01, 0xffff0101ffffffff, ++ 0x01ff01ff01010101, 0xffff01ff010101ff, 0xffff01ff0101ff01, 0x01ff01ff0101ffff, ++ 0xffff01ff01ff0101, 0x01ff01ff01ff01ff, 0x01ff01ff01ffff01, 0xffff01ff01ffffff, ++ 0xffff01ffff010101, 0x01ff01ffff0101ff, 0x01ff01ffff01ff01, 0xffff01ffff01ffff, ++ 0x01ff01ffffff0101, 0xffff01ffffff01ff, 0xffff01ffffffff01, 0x01ff01ffffffffff, ++ 0x01ffff0101010101, 0xffffff01010101ff, 0xffffff010101ff01, 0x01ffff010101ffff, ++ 0xffffff0101ff0101, 0x01ffff0101ff01ff, 0x01ffff0101ffff01, 0xffffff0101ffffff, ++ 0xffffff01ff010101, 0x01ffff01ff0101ff, 0x01ffff01ff01ff01, 0xffffff01ff01ffff, ++ 0x01ffff01ffff0101, 0xffffff01ffff01ff, 0xffffff01ffffff01, 0x01ffff01ffffffff, ++ 0xffffffff01010101, 0x01ffffff010101ff, 0x01ffffff0101ff01, 0xffffffff0101ffff, ++ 0x01ffffff01ff0101, 0xffffffff01ff01ff, 0xffffffff01ffff01, 0x01ffffff01ffffff, ++ 0x01ffffffff010101, 0xffffffffff0101ff, 0xffffffffff01ff01, 0x01ffffffff01ffff, ++ 0xffffffffffff0101, 0x01ffffffffff01ff, 0x01ffffffffffff01, 0xffffffffffffffff, ++}; ++ ++#ifdef __AVX2__ ++ ++#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1) ++#define MM256_SET1_M128I(x) _mm256_broadcastsi128_si256(x) ++ ++static inline float hsum_float_4(__m128 x) { ++ x = _mm_add_ps(x, _mm_movehl_ps(x, x)); ++ x = _mm_add_ss(x, _mm_movehdup_ps(x)); ++ return _mm_cvtss_f32(x); ++} ++static inline float hsum_float_8(__m256 x) { ++ return hsum_float_4(_mm_add_ps(_mm256_castps256_ps128(x), _mm256_extractf128_ps(x, 1))); ++} ++ ++static inline __m128i load_iq4nl_values_128() { ++ // kvalues_iq4nl + 128 ++ static const uint8_t k_values[16] = {1, 24, 45, 63, 79, 93, 106, 118, 129, 141, 153, 166, 181, 197, 217, 241}; ++ return _mm_loadu_si128((const __m128i *)k_values); ++} ++ ++static inline __m256i load_iq4nl_values_256() { ++ auto val128 = load_iq4nl_values_128(); ++ return MM256_SET1_M128I(val128); ++} ++ ++#ifdef HAVE_FANCY_SIMD ++static inline __m512i load_iq4nl_values_512() { ++ auto val256 = load_iq4nl_values_256(); ++ return _mm512_inserti32x8(_mm512_castsi256_si512(val256), val256, 1); ++} ++#endif ++ ++template struct Q8 { ++ ++ constexpr static int nrc_y = nrc; ++ ++ Q8(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8 *)info.src1_row(iy); ++ } ++ ++#ifdef HAVE_FANCY_SIMD ++ inline __m512i load_quants64(int iy, int i, int j) const { return _mm512_loadu_si512((const __m512i*)y[iy][i].qs + j); } ++#endif ++ inline __m256i load_quants(int iy, int i, int j) const { return _mm256_loadu_si256((const __m256i*)y[iy][i].qs + j); } ++ inline __m256i load_bsums(int iy, int i) const { return _mm256_loadu_si256((const __m256i*)y[iy][i].bsums); } ++ inline float scale(int iy, int i) const { return y[iy][i].d; } ++ ++ const block_q8 * y[nrc_y]; ++}; ++ ++struct Scales8KBase { ++ template ++ inline void accum_mins(const __m128i& mins128, const Q8& q8, int i, float c, __m256 * accd) const { ++ const __m256i mins = MM256_SET_M128I(_mm_shuffle_epi8(mins128, shuffles[1]), _mm_shuffle_epi8(mins128, shuffles[0])); ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i q8s = q8.load_bsums(iy, i); ++ const __m256i prod = _mm256_madd_epi16(mins, q8s); ++ accd[iy] = _mm256_fmadd_ps(_mm256_set1_ps(c*q8.scale(iy, i)), _mm256_cvtepi32_ps(prod), accd[iy]); ++ } ++ } ++ inline __m256i shuffle(__m128i mins) const { ++ return MM256_SET_M128I(_mm_shuffle_epi8(mins, shuffles[1]), _mm_shuffle_epi8(mins, shuffles[0])); ++ } ++ const __m128i shuffles[2] = {_mm_set_epi32(0x07060706, 0x05040504, 0x03020302, 0x01000100), ++ _mm_set_epi32(0x0f0e0f0e, 0x0d0c0d0c, 0x0b0a0b0a, 0x09080908)}; ++}; ++ ++template ++struct BaseDequantizer { ++ BaseDequantizer(const void * vx, size_t bx) : vx(vx), bx(bx) {} ++ inline void new_row(int ix) { ++ if constexpr (per_row_scale) { ++ if constexpr (is_f16) { ++ const ggml_half * dptr = (const ggml_half *)((const char *)vx + bx*ix); ++ d = GGML_FP16_TO_FP32(*dptr); ++ x = (const Block *)(dptr + 1); ++ } else { ++ const float * dptr = (const float *)((const char *)vx + bx*ix); ++ d = *dptr; ++ x = (const Block *)(dptr + 1); ++ } ++ } else { ++ x = (const Block *)((const char *)vx + bx*ix); ++ } ++ } ++ ++ const void * vx; ++ const size_t bx; ++ const Block * x; ++ ++ float d; ++}; ++ ++template ++static inline void multiply_add(const Bits& bits, const __m256i * scales, int j, int i, const Q8& q8, __m256i * sumi) { ++ if (j == 0) { ++#ifdef HAVE_FANCY_SIMD ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ sumi[iy] = _mm256_dpwssd_epi32(_mm256_setzero_si256(), scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 0))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 1))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 2))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 3))); ++ } ++#else ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i p1 = _mm256_madd_epi16(scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 0))); ++ const __m256i p2 = _mm256_madd_epi16(scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 1))); ++ const __m256i p3 = _mm256_madd_epi16(scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 2))); ++ const __m256i p4 = _mm256_madd_epi16(scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 3))); ++ sumi[iy] = _mm256_add_epi32(_mm256_add_epi32(p1, p3), _mm256_add_epi32(p2, p4)); ++ } ++#endif ++ } else { ++#ifdef HAVE_FANCY_SIMD ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 4))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 5))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 6))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 7))); ++ } ++#else ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i p1 = _mm256_madd_epi16(scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 4))); ++ const __m256i p2 = _mm256_madd_epi16(scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 5))); ++ const __m256i p3 = _mm256_madd_epi16(scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 6))); ++ const __m256i p4 = _mm256_madd_epi16(scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 7))); ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p1, p3)); ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p2, p4)); ++ } ++#endif ++ } ++} ++ ++template ++static inline void multiply_add_avx2(const Bits& bits, const __m256i * scales, int j, int i, const Q8& q8, __m256i * sumi) { ++ __m256i p[4]; ++ if (j == 0) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ for (int k = 0; k < 4; ++k) { ++ auto s = _mm256_sign_epi8(bits.values[k], bits.values[k]); ++ p[k] = _mm256_madd_epi16(scales[k], _mm256_maddubs_epi16(s, _mm256_sign_epi8(q8.load_quants(iy, i, k), bits.values[k]))); ++ } ++ sumi[iy] = _mm256_add_epi32(_mm256_add_epi32(p[0], p[1]), _mm256_add_epi32(p[2], p[3])); ++ } ++ } else { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ for (int k = 0; k < 4; ++k) { ++ auto s = _mm256_sign_epi8(bits.values[k], bits.values[k]); ++ p[k] = _mm256_madd_epi16(scales[k], _mm256_maddubs_epi16(s, _mm256_sign_epi8(q8.load_quants(iy, i, 4+k), bits.values[k]))); ++ } ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p[0], p[2])); ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p[1], p[3])); ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++ ++struct BlockPermuter { ++ const __m512i permute1 = _mm512_set_epi64(11, 10, 9, 8, 3, 2, 1, 0); ++ const __m512i permute2 = _mm512_set_epi64(15, 14, 13, 12, 7, 6, 5, 4); ++}; ++ ++struct Q4Bits { ++ inline void prepare(const uint8_t * q4) { ++ auto q4bits = _mm512_loadu_si512((const __m512i*)q4 + 0); ++ auto tmp1 = _mm512_and_si512(q4bits, ml); ++ auto tmp2 = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ values[0] = _mm512_permutex2var_epi64(tmp1, perm.permute1, tmp2); ++ values[1] = _mm512_permutex2var_epi64(tmp1, perm.permute2, tmp2); ++ q4bits = _mm512_loadu_si512((const __m512i*)q4 + 1); ++ tmp1 = _mm512_and_si512(q4bits, ml); ++ tmp2 = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ values[2] = _mm512_permutex2var_epi64(tmp1, perm.permute1, tmp2); ++ values[3] = _mm512_permutex2var_epi64(tmp1, perm.permute2, tmp2); ++ } ++ inline void prepare64(const uint8_t * q4) { ++ auto q4bits = _mm512_loadu_si512((const __m512i*)q4 + 0); ++ values[0] = _mm512_and_si512(q4bits, ml); ++ values[1] = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ q4bits = _mm512_loadu_si512((const __m512i*)q4 + 1); ++ values[2] = _mm512_and_si512(q4bits, ml); ++ values[3] = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ } ++ inline void prepare64a(const uint8_t * q4) { ++ for (int k = 0; k < 4; ++k) { ++ auto q4bits = _mm256_loadu_si256((const __m256i*)q4 + k); ++ values[k] = _mm512_inserti32x8(_mm512_castsi256_si512(q4bits), _mm256_srli_epi16(q4bits, 4), 1); ++ values[k] = _mm512_and_si512(values[k], ml); ++ } ++ } ++ __m512i values[4]; ++ const __m512i ml = _mm512_set1_epi8(0xf); ++ const BlockPermuter perm; ++}; ++ ++inline __m512i dpbusd_permd() { ++ return _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); ++} ++inline __m512 dpbusd_scales(float d, __m256i scales16) { ++ auto vd = _mm512_mul_ps(_mm512_set1_ps(d), _mm512_cvtepi32_ps(_mm512_cvtepi16_epi32(scales16))); ++ return _mm512_permutex2var_ps(vd, dpbusd_permd(), vd); ++} ++inline __m512i dpbusd_permute_epi16(__m256i v) { ++ return _mm512_maskz_permutexvar_epi16(0x55555555, dpbusd_permd(), _mm512_castsi256_si512(v)); ++} ++ ++template ++static void mul_mat_iqX_k_q8_K_AVX512_dpbusd(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ __m512 accd[nrc_y] = {}; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ __m512i mins; ++ auto d4 = deq.new_block(i, mins); ++ ++ __m512i p[4]; ++#pragma GCC unroll 8 ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ for (int k = 0; k < 4; k += 2) { ++ auto y1 = q8.load_quants64(iy, i, k+0); ++ auto y2 = q8.load_quants64(iy, i, k+1); ++ p[k+0] = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[k+0], y1); ++ p[k+1] = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[k+1], y2); ++ } ++ auto bsums = dpbusd_permute_epi16(q8.load_bsums(iy, i)); ++ auto d48 = _mm512_mul_ps(d4, _mm512_set1_ps(q8.scale(iy, i))); ++ auto p01 = _mm512_add_epi32(_mm512_unpacklo_epi32(p[0], p[1]), _mm512_unpackhi_epi32(p[0], p[1])); ++ auto p23 = _mm512_add_epi32(_mm512_unpacklo_epi32(p[2], p[3]), _mm512_unpackhi_epi32(p[2], p[3])); ++ auto sum = _mm512_add_epi32(_mm512_unpacklo_epi64(p01, p23), _mm512_unpackhi_epi64(p01, p23)); ++ sum = _mm512_dpwssd_epi32(sum, mins, bsums); ++ accd[iy] = _mm512_fmadd_ps(d48, _mm512_cvtepi32_ps(sum), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, _mm512_reduce_add_ps(accd[iy])); ++ accd[iy] = _mm512_setzero_ps(); ++ } ++ ++ } ++} ++ ++struct Q2Bits { ++ inline void prepare(const uint8_t * q2) { ++ ++ auto q2bits = _mm512_loadu_si512((const __m512i*)q2); ++ auto tmp = _mm512_srli_epi16(q2bits, 2); ++ ++ values[0] = _mm512_permutex2var_epi64(q2bits, perm.permute1, tmp); ++ values[2] = _mm512_permutex2var_epi64(q2bits, perm.permute2, tmp); ++ values[1] = _mm512_and_si512(_mm512_srli_epi16(values[0], 4), ml); ++ values[3] = _mm512_and_si512(_mm512_srli_epi16(values[2], 4), ml); ++ values[0] = _mm512_and_si512(values[0], ml); ++ values[2] = _mm512_and_si512(values[2], ml); ++ } ++ __m512i values[4]; ++ const __m512i ml = _mm512_set1_epi8(0x03); ++ BlockPermuter perm; ++}; ++ ++#else ++ ++struct Q2Bits { ++ inline void prepare(const uint8_t * q2, int j) { ++ auto q2bits = _mm256_loadu_si256((const __m256i *)q2 + j); ++ values[0] = _mm256_and_si256(q2bits, ml); ++ values[1] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), ml); ++ values[2] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), ml); ++ values[3] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), ml); ++ } ++ __m256i values[4]; ++ const __m256i ml = _mm256_set1_epi8(0x03); ++}; ++ ++struct Q4Bits { ++ inline void prepare(const uint8_t * q4, int j) { ++ auto q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+0); ++ values[0] = _mm256_and_si256(q4bits, ml); ++ values[1] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+1); ++ values[2] = _mm256_and_si256(q4bits, ml); ++ values[3] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ } ++ inline void prepare64(const uint8_t * q4, int j) { ++ auto q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+0); ++ values[0] = _mm256_and_si256(q4bits, ml); ++ values[2] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+1); ++ values[1] = _mm256_and_si256(q4bits, ml); ++ values[3] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ } ++ inline void prepare16(const uint8_t * q4, int j) { ++ values[0] = dequant16(q4 + 64*j + 0); ++ values[1] = dequant16(q4 + 64*j + 16); ++ values[2] = dequant16(q4 + 64*j + 32); ++ values[3] = dequant16(q4 + 64*j + 48); ++ } ++ inline __m256i dequant16(const uint8_t * qs) const { ++ const __m128i aux128 = _mm_loadu_si128((const __m128i *)qs); ++ const __m256i aux256 = MM256_SET_M128I(_mm_srli_epi16(aux128, 4), aux128); ++ return _mm256_and_si256(ml, aux256); ++ } ++ __m256i values[4]; ++ const __m256i ml = _mm256_set1_epi8(0xf); ++}; ++ ++#endif ++ ++#else ++// ------------------------------------ __aarch64__ -------------------------------------------------- ++ ++template struct Q8 { ++ ++ constexpr static int nrc_y = nrc; ++ ++ Q8(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8 *)info.src1_row(iy); ++ } ++ ++ inline int8x16x2_t load_quants(int iy, int i, int j) const { return vld1q_s8_x2(y[iy][i].qs + 32*j); } ++ inline int8x16x4_t load_quants_64(int iy, int i, int j) const { return vld1q_s8_x4(y[iy][i].qs + 64*j); } ++ inline int16x8x2_t load_bsums(int iy, int i) const { return vld1q_s16_x2(y[iy][i].bsums); } ++ inline int16x8_t load_bsums8(int iy, int i) const { ++ auto q8s = vld1q_s16_x2(y[iy][i].bsums); ++ return vpaddq_s16(q8s.val[0], q8s.val[1]); ++ } ++ inline float scale(int iy, int i) const { return y[iy][i].d; } ++ ++ const block_q8 * y[nrc_y]; ++}; ++ ++template ++struct BaseDequantizer { ++ BaseDequantizer(const void * vx, size_t bx, int nrc) : vx(vx), x(nullptr), bx(bx), nrc(nrc) {} ++ inline void new_row(int ix) { ++ if constexpr (has_row_scale) { ++ if constexpr (scale_is_f16) { ++ const ggml_half * dptr = (const ggml_half *)((const char *)vx + ix*bx); ++ d = GGML_FP16_TO_FP32(*dptr); ++ x = (const block_q *)(dptr + 1); ++ } else { ++ const float * dptr = (const float *)((const char *)vx + ix*bx); ++ d = *dptr; ++ x = (const block_q *)(dptr + 1); ++ } ++ } else { ++ x = (const block_q *)((const char *)vx + ix*bx); ++ } ++ } ++ const void * vx; ++ const block_q * x; ++ const size_t bx; ++ const int nrc; ++ float d; ++}; ++ ++struct Q4bits { ++ const uint8x16_t m4b = vdupq_n_u8(0xf); ++ uint8x16x4_t b1, b2; ++ inline void prepare4(uint8x16x4_t& b, const uint8x16_t * val) const { ++ b.val[0] = vandq_u8(val[0], m4b); ++ b.val[2] = vshrq_n_u8(val[0], 4); ++ b.val[1] = vandq_u8(val[1], m4b); ++ b.val[3] = vshrq_n_u8(val[1], 4); ++ } ++ inline void prepare4_16(uint8x16x4_t& b, const uint8x16_t * val) const { ++ b.val[0] = vandq_u8(val[0], m4b); ++ b.val[1] = vshrq_n_u8(val[0], 4); ++ b.val[2] = vandq_u8(val[1], m4b); ++ b.val[3] = vshrq_n_u8(val[1], 4); ++ } ++ inline void prepare(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x2(qs); ++ prepare4(b1, q4bits.val); ++ q4bits = vld1q_u8_x2(qs+32); ++ prepare4(b2, q4bits.val); ++ } ++ inline void prepare_v2(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x4(qs); ++ prepare4(b1, q4bits.val+0); ++ prepare4(b2, q4bits.val+2); ++ } ++ inline void prepare64(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x4(qs); ++ b1.val[0] = vandq_u8(q4bits.val[0], m4b); ++ b1.val[1] = vandq_u8(q4bits.val[1], m4b); ++ b1.val[2] = vandq_u8(q4bits.val[2], m4b); ++ b1.val[3] = vandq_u8(q4bits.val[3], m4b); ++ b2.val[0] = vshrq_n_u8(q4bits.val[0], 4); ++ b2.val[1] = vshrq_n_u8(q4bits.val[1], 4); ++ b2.val[2] = vshrq_n_u8(q4bits.val[2], 4); ++ b2.val[3] = vshrq_n_u8(q4bits.val[3], 4); ++ } ++ inline void prepare16(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x2(qs); ++ prepare4_16(b1, q4bits.val); ++ q4bits = vld1q_u8_x2(qs+32); ++ prepare4_16(b2, q4bits.val); ++ } ++ inline void prepare16_v2(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x4(qs); ++ prepare4_16(b1, q4bits.val+0); ++ prepare4_16(b2, q4bits.val+2); ++ } ++}; ++ ++struct Q2bits { ++ const uint8x16_t m4b = vdupq_n_u8(0x03); ++ uint8x16x4_t b1, b2; ++ inline void prepare(const uint8_t * qs) { ++ auto q2bits = vld1q_u8_x2(qs); ++ b1.val[0] = vandq_u8(q2bits.val[0], m4b); ++ b1.val[1] = vandq_u8(q2bits.val[1], m4b); ++ ++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2); ++ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2); ++ b1.val[2] = vandq_u8(q2bits.val[0], m4b); ++ b1.val[3] = vandq_u8(q2bits.val[1], m4b); ++ ++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2); ++ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2); ++ b2.val[0] = vandq_u8(q2bits.val[0], m4b); ++ b2.val[1] = vandq_u8(q2bits.val[1], m4b); ++ ++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2); ++ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2); ++ b2.val[2] = vandq_u8(q2bits.val[0], m4b); ++ b2.val[3] = vandq_u8(q2bits.val[1], m4b); ++ } ++}; ++ ++template ++static inline void compute_8_blocks(const uint8x16x4_t& qx_1, const uint8x16x4_t& qx_2, const Q8& q8, ++ const int32x4x2_t& scales, int iy, int i, int j, int32x4_t& sumi) { ++ auto mzero = vdupq_n_s32(0); ++ auto q8b_1 = q8.load_quants(iy, i, 4*j+0); ++ auto p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[0]), q8b_1.val[0]), ++ vreinterpretq_s8_u8(qx_1.val[1]), q8b_1.val[1]); // block 1 ++ auto q8b_2 = q8.load_quants(iy, i, 4*j+1); ++ auto p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[2]), q8b_2.val[0]), ++ vreinterpretq_s8_u8(qx_1.val[3]), q8b_2.val[1]); // block 2 ++ auto p12 = vpaddq_s32(p1, p2); ++ ++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2); ++ auto p3 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[0]), q8b_3.val[0]), ++ vreinterpretq_s8_u8(qx_2.val[1]), q8b_3.val[1]); // block 1 ++ auto q8b_4 = q8.load_quants(iy, i, 4*j+3); ++ auto p4 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[2]), q8b_4.val[0]), ++ vreinterpretq_s8_u8(qx_2.val[3]), q8b_4.val[1]); // block 2 ++ auto p34 = vpaddq_s32(p3, p4); ++ ++ auto pall = vpaddq_s32(p12, p34); ++ sumi = vmlaq_s32(sumi, scales.val[j], pall); ++} ++ ++template ++static inline void compute_16_blocks(const uint8x16x4_t& qx_1, const uint8x16x4_t& qx_2, const Q8& q8, ++ const int32x4x4_t& scales, int iy, int i, int j, int32x4_t& sumi) { ++ ++ auto mzero = vdupq_n_s32(0); ++ auto q8b_1 = q8.load_quants(iy, i, 4*j+0); ++ auto p1 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[0]), q8b_1.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[1]), q8b_1.val[1])); // blocks 0, 0, 1, 1, ++ auto q8b_2 = q8.load_quants(iy, i, 4*j+1); ++ auto p2 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[2]), q8b_2.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[3]), q8b_2.val[1])); // blocks 3, 3, 4, 4, ++ auto p12 = vpaddq_s32(p1, p2); // blocks 0, 1, 2, 3 ++ sumi = vmlaq_s32(sumi, scales.val[2*j+0], p12); ++ ++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2); ++ auto p3 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[0]), q8b_3.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[1]), q8b_3.val[1])); // block 4, 4, 5, 5, ++ auto q8b_4 = q8.load_quants(iy, i, 4*j+3); ++ auto p4 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[2]), q8b_4.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[3]), q8b_4.val[1])); // block 6, 6, 7, 7, ++ auto p34 = vpaddq_s32(p3, p4); // blocks 4, 5, 6, 7 ++ sumi = vmlaq_s32(sumi, scales.val[2*j+1], p34); ++} ++ ++struct SignHelper { ++ ++ inline void init() { shuffle = vcombine_u8(vdup_n_u8(0), vdup_n_u8(1)); } ++ ++ inline void apply_signs_1(uint8x16_t * b, const uint8x16_t& signs16) { ++ auto aux = vqtbl1q_u8(signs16, shuffle); ++ auto s = vreinterpretq_s8_u8(vorrq_u8(vceqq_u8(vandq_u8(aux, smask), smask), m1)); ++ b[0] = vreinterpretq_u8_s8(vmulq_s8(vreinterpretq_s8_u8(b[0]), s)); ++ shuffle = vaddq_u8(shuffle, step); ++ } ++ ++ const uint8x16_t smask = vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201)); ++ const uint8x16_t m1 = vdupq_n_u8(1); ++ const uint8x16_t step = vdupq_n_u8(2); ++ uint8x16_t shuffle; ++}; ++ ++template ++static void mul_mat_qX_K_q8_K_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx, nrc_y); ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ float32x4_t acc[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ int32x4_t sumi[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) sumi[iy] = vdupq_n_s32(0); ++ ++ if constexpr (nrc_y > 1 && Dequantizer::should_scale_quants()) { ++ deq.process_scales(i, q8, acc); ++ deq.prepare(i, 0); ++ deq.compute(q8, i, 0, sumi); ++ deq.prepare(i, 1); ++ deq.compute(q8, i, 1, sumi); ++ } else { ++ if constexpr (Dequantizer::num_blocks() == 8) { ++ auto scales = deq.new_block(i, q8, acc); ++ deq.prepare(i, 0); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_8_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 0, sumi[iy]); ++ deq.prepare(i, 1); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_8_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 1, sumi[iy]); ++ } ++ else if constexpr (Dequantizer::num_blocks() == 16) { ++ auto scales = deq.new_block(i, q8, acc); ++ deq.prepare(i, 0); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_16_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 0, sumi[iy]); ++ deq.prepare(i, 1); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_16_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 1, sumi[iy]); ++ } ++ else { ++ GGML_ASSERT(false); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = vmlaq_f32(acc[iy], vcvtq_f32_s32(sumi[iy]), vdupq_n_f32(deq.d*q8.scale(iy, i))); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, vaddvq_f32(acc[iy])); ++ } ++ } ++} ++ ++#endif ++ ++#endif // IQK_IMPLEMENT +diff --git a/src/ggml-cpu/iqk/iqk_gemm.cpp b/src/ggml-cpu/iqk/iqk_gemm.cpp +new file mode 100644 +index 00000000..f82a1c5e +--- /dev/null ++++ b/src/ggml-cpu/iqk/iqk_gemm.cpp +@@ -0,0 +1,224 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++// Adapted from ik_llama.cpp (https://github.com/ikawrakow/ik_llama.cpp) commit 009dce19, ++// ggml/src/iqk/{iqk_config.h, iqk_common.h, iqk_gemm_kquants.cpp, iqk_gemm_iquants.cpp, iqk_gemm_1bit.cpp, iqk_mul_mat.cpp}. ++// Only the kernels that read the upstream ggml block layouts with upstream Q8_K activations are kept. ++// ++// Permission is hereby granted, free of charge, to any person obtaining a copy ++// of this software and associated documentation files (the "Software"), to deal ++// in the Software without restriction, including without limitation the rights ++// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell ++// copies of the Software, and to permit persons to whom the Software is ++// furnished to do so, subject to the following conditions: ++// ++// The above copyright notice and this permission notice shall be included in all ++// copies or substantial portions of the Software. ++// ++// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ++// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++// SOFTWARE. ++// ++// Driver of the iqk GEMM kernels (the tiling of MulMat::mul_mat_NxM in iqk_mul_mat.cpp). ++ ++#include "iqk_gemm.h" ++#include "iqk_common.h" ++ ++#include "ggml.h" ++ ++#include ++#include ++#include ++#include ++ ++namespace { ++ ++// the types this file can serve (whether or not this CPU variant has kernels for them) ++constexpr uint64_t iqk_type_bit(ggml_type t) { return 1ull << (int) t; } ++[[maybe_unused]] constexpr uint64_t k_iqk_all_types = ++ iqk_type_bit(GGML_TYPE_Q2_K) | iqk_type_bit(GGML_TYPE_Q3_K) | iqk_type_bit(GGML_TYPE_Q4_K) | ++ iqk_type_bit(GGML_TYPE_Q5_K) | iqk_type_bit(GGML_TYPE_Q6_K) | iqk_type_bit(GGML_TYPE_IQ4_XS) | ++ iqk_type_bit(GGML_TYPE_IQ2_XXS) | iqk_type_bit(GGML_TYPE_IQ2_XS) | iqk_type_bit(GGML_TYPE_IQ2_S) | ++ iqk_type_bit(GGML_TYPE_IQ3_XXS) | iqk_type_bit(GGML_TYPE_IQ3_S) | iqk_type_bit(GGML_TYPE_IQ1_S) | ++ iqk_type_bit(GGML_TYPE_IQ1_M); ++ ++#ifdef IQK_IMPLEMENT ++constexpr uint64_t k_iqk_built_types = k_iqk_all_types; ++#else ++constexpr uint64_t k_iqk_built_types = 0; ++#endif ++ ++std::atomic g_iqk_types{k_iqk_built_types}; ++std::atomic g_iqk_count[64]; ++int g_iqk_debug = 0; ++bool g_iqk_first = false; ++ ++#ifdef IQK_IMPLEMENT ++ ++bool iqk_set_kernels(int type, mul_mat_t * funcs) { ++ for (int i = 0; i < IQK_MAX_NY; ++i) funcs[i] = nullptr; ++ return iqk_set_kernels_kquants(type, funcs) || iqk_set_kernels_iquants(type, funcs) || iqk_set_kernels_1bit(type, funcs); ++} ++ ++// rows [0, nrc_x) of vx times the activation rows [info.cur_y, nrc_y) ++void iqk_mul_mat_NxM(const mul_mat_t * funcs, int n, const void * vx, size_t bx, DataInfo & info, int nrc_x, int nrc_y) { ++ constexpr int k_x_step = 64; ++ int ny = IQK_MAX_NY; ++ while (ny > 0 && !funcs[ny-1]) --ny; ++ int n_left = nrc_y - info.cur_y; ++ int n_step = n_left/ny; ++ if (n_step > 0) { ++ if (n_step*ny != n_left) { ++ ++n_step; ++ int ny1 = n_left/n_step; ++ int ny2 = ny1 + 1; ++ int my1 = n_step*ny2 - n_left; ++ int my2 = n_step - my1; ++ for (int ix = 0; ix < nrc_x; ix += k_x_step) { ++ auto this_info = info; ++ this_info.s += ix; ++ int this_nrc_x = ix + k_x_step <= nrc_x ? k_x_step : nrc_x - ix; ++ for (int iy = 0; iy < my1; ++iy) { ++ funcs[ny1-1](n, (const void *)((const char *)vx + ix*bx), bx, this_info, this_nrc_x); ++ this_info.cur_y += ny1; ++ } ++ for (int iy = 0; iy < my2; ++iy) { ++ funcs[ny2-1](n, (const void *)((const char *)vx + ix*bx), bx, this_info, this_nrc_x); ++ this_info.cur_y += ny2; ++ } ++ } ++ info.cur_y += n_left; ++ } ++ else { ++ for (int ix = 0; ix < nrc_x; ix += k_x_step) { ++ auto this_info = info; ++ this_info.s += ix; ++ int this_nrc_x = ix + k_x_step <= nrc_x ? k_x_step : nrc_x - ix; ++ for (int iy = 0; iy < n_step; ++iy) { ++ funcs[ny-1](n, (const void *)((const char *)vx + ix*bx), bx, this_info, this_nrc_x); ++ this_info.cur_y += ny; ++ } ++ } ++ info.cur_y += ny * n_step; ++ } ++ } ++ n_left = nrc_y - info.cur_y; ++ if (n_left > 0) { ++ funcs[n_left-1](n, vx, bx, info, nrc_x); ++ } ++} ++ ++#endif // IQK_IMPLEMENT ++ ++} // namespace ++ ++extern "C" { ++ ++void ggml_cpu_iqk_init(void) { ++ uint64_t mask = k_iqk_built_types; ++ const char * on = getenv("GGML_CPU_IQK"); ++ if (on != NULL && atoi(on) == 0) { ++ mask = 0; ++ } ++ g_iqk_first = on != NULL && atoi(on) == 2; ++ const char * env = getenv("GGML_CPU_IQK_TYPES"); ++ if (mask != 0 && env != NULL && strcmp(env, "all") != 0) { ++ uint64_t sel = 0; ++ char buf[256]; ++ snprintf(buf, sizeof(buf), "%s", env); ++ for (char * tok = strtok(buf, ", "); tok != NULL; tok = strtok(NULL, ", ")) { ++ for (int t = 0; t < GGML_TYPE_COUNT && t < 64; ++t) { ++ const char * name = ggml_type_name((enum ggml_type) t); ++ if (name != NULL && strcmp(name, tok) == 0) { ++ sel |= 1ull << t; ++ } ++ } ++ } ++ mask &= sel; ++ } ++ g_iqk_types.store(mask); ++ const char * dbg = getenv("GGML_CPU_IQK_DEBUG"); ++ g_iqk_debug = dbg != NULL ? atoi(dbg) : 0; ++ if (g_iqk_debug) { ++ fprintf(stderr, "ggml-cpu iqk gemm: built types 0x%llx, enabled 0x%llx (%s the tiled path):", (unsigned long long) k_iqk_built_types, ++ (unsigned long long) mask, g_iqk_first ? "ahead of" : "after"); ++ for (int t = 0; t < GGML_TYPE_COUNT && t < 64; ++t) { ++ if ((mask >> t) & 1) fprintf(stderr, " %s", ggml_type_name((enum ggml_type) t)); ++ } ++ fprintf(stderr, "\n"); ++ } ++} ++ ++bool ggml_cpu_iqk_type_enabled(int type) { ++ return type >= 0 && type < 64 && ((g_iqk_types.load(std::memory_order_relaxed) >> type) & 1); ++} ++ ++bool ggml_cpu_iqk_before_tiled(void) { ++ return g_iqk_first; ++} ++ ++int64_t ggml_cpu_iqk_count(int type) { ++ return type >= 0 && type < 64 ? g_iqk_count[type].load() : 0; ++} ++ ++void ggml_cpu_iqk_set_types(uint64_t mask) { ++ g_iqk_types.store(mask & k_iqk_built_types); ++} ++ ++bool ggml_cpu_iqk_mul_mat(int type, int64_t Nx, int64_t Ny, int64_t ne00, ++ const void * A, size_t stride_A, const void * B, size_t stride_B, ++ float * C, size_t stride_C, int ith, int nth) { ++#ifdef IQK_IMPLEMENT ++ if (ne00 % QK_K != 0 || Nx <= 0 || Ny <= 0 || Nx > INT32_MAX || Ny > INT32_MAX || ne00 > INT32_MAX) { ++ return false; ++ } ++ mul_mat_t funcs[IQK_MAX_NY]; ++ if (!iqk_set_kernels(type, funcs)) { ++ return false; ++ } ++ if (ith == 0) { ++ const int64_t prev = g_iqk_count[type].fetch_add(1, std::memory_order_relaxed); ++ if (g_iqk_debug && prev == 0) { ++ fprintf(stderr, "ggml-cpu iqk gemm: first %s mul_mat (%lld x %lld x %lld, %d threads)\n", ++ ggml_type_name((enum ggml_type) type), (long long) Nx, (long long) Ny, (long long) ne00, nth); ++ } ++ } ++ ++ // Tiles of rows x activation columns: the rows are split over the threads first (each thread then walks all ++ // columns of its row range, 64 rows x up to 8 columns per kernel call); with fewer than 16 rows per thread the ++ // columns are split too. ++ const int nx = (int) Nx; ++ const int ny = (int) Ny; ++ int ntile_x = nth < nx/16 ? nth : (nx/16 > 0 ? nx/16 : 1); ++ int ntile_y = nth/ntile_x; ++ if (ntile_y > (ny + 7)/8) ntile_y = (ny + 7)/8; ++ if (ntile_y < 1) ntile_y = 1; ++ const int ntile = ntile_x*ntile_y; ++ for (int itile = ith; itile < ntile; itile += nth) { ++ const int jx = itile % ntile_x; ++ const int jy = itile / ntile_x; ++ const int ix0 = (int)((int64_t) jx * nx / ntile_x); ++ const int ix1 = (int)((int64_t)(jx + 1) * nx / ntile_x); ++ const int iy0 = (int)((int64_t) jy * ny / ntile_y); ++ const int iy1 = (int)((int64_t)(jy + 1) * ny / ntile_y); ++ if (ix1 <= ix0 || iy1 <= iy0) { ++ continue; ++ } ++ DataInfo info{C + ix0, (const char *) B, stride_C, stride_B, iy0}; ++ iqk_mul_mat_NxM(funcs, (int) ne00, (const char *) A + ix0*stride_A, stride_A, info, ix1 - ix0, iy1); ++ } ++ return true; ++#else ++ (void) type; (void) Nx; (void) Ny; (void) ne00; (void) A; (void) stride_A; (void) B; (void) stride_B; ++ (void) C; (void) stride_C; (void) ith; (void) nth; ++ return false; ++#endif ++} ++ ++} // extern "C" +diff --git a/src/ggml-cpu/iqk/iqk_gemm.h b/src/ggml-cpu/iqk/iqk_gemm.h +new file mode 100644 +index 00000000..466e942e +--- /dev/null ++++ b/src/ggml-cpu/iqk/iqk_gemm.h +@@ -0,0 +1,40 @@ ++#pragma once ++ ++// GEMM kernels ported from ik_llama.cpp (iqk_gemm.cpp) for the k-quant / i-quant weight types whose ++// activations are Q8_K. They read the upstream block layouts and the upstream Q8_K activation rows and ++// form the same integer products as the ggml vec_dot kernels; only the float accumulation order differs. ++ ++#include ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++// Reads once: GGML_CPU_IQK (0 disables; 2 runs these kernels ahead of the x86 tiled K-quant path instead of after it), ++// GGML_CPU_IQK_TYPES (ggml type names or "all"; the other types keep the tiled / vec_dot paths) and GGML_CPU_IQK_DEBUG ++// (1: print the enabled types and the first mul_mat per type to stderr). ++void ggml_cpu_iqk_init(void); ++ ++// true if this build has a kernel for the weight type and it is enabled ++bool ggml_cpu_iqk_type_enabled(int type); ++ ++// GGML_CPU_IQK=2: the iqk kernels take the op before the tiled path gets a chance ++bool ggml_cpu_iqk_before_tiled(void); ++ ++// C[iy*stride_C + ix] = dot(A row ix, B row iy) for ix < Nx, iy < Ny; A rows of type `type` (stride_A bytes), ++// B rows are Q8_K (stride_B bytes), ne00 % 256 == 0. The work is split over the nth threads by ith. ++// Returns false (computing nothing) if the type is not handled. ++bool ggml_cpu_iqk_mul_mat(int type, int64_t Nx, int64_t Ny, int64_t ne00, ++ const void * A, size_t stride_A, const void * B, size_t stride_B, ++ float * C, size_t stride_C, int ith, int nth); ++ ++// test hooks (reached through the CPU backend's get_proc_address): number of mul_mat ops computed by ++// these kernels per weight type, and an override of the enabled type mask (bit = ggml type) ++int64_t ggml_cpu_iqk_count(int type); ++void ggml_cpu_iqk_set_types(uint64_t mask); ++ ++#ifdef __cplusplus ++} ++#endif +diff --git a/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp b/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp +new file mode 100644 +index 00000000..f9596cfb +--- /dev/null ++++ b/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp +@@ -0,0 +1,364 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++// Adapted from ik_llama.cpp (https://github.com/ikawrakow/ik_llama.cpp) commit 009dce19, ++// ggml/src/iqk/{iqk_config.h, iqk_common.h, iqk_gemm_kquants.cpp, iqk_gemm_iquants.cpp, iqk_gemm_1bit.cpp, iqk_mul_mat.cpp}. ++// Only the kernels that read the upstream ggml block layouts with upstream Q8_K activations are kept. ++// ++// Permission is hereby granted, free of charge, to any person obtaining a copy ++// of this software and associated documentation files (the "Software"), to deal ++// in the Software without restriction, including without limitation the rights ++// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell ++// copies of the Software, and to permit persons to whom the Software is ++// furnished to do so, subject to the following conditions: ++// ++// The above copyright notice and this permission notice shall be included in all ++// copies or substantial portions of the Software. ++// ++// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ++// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++// SOFTWARE. ++// ++// IQ1_S and IQ1_M times Q8_K (from iqk_gemm_1bit.cpp). ++ ++#include "iqk_common.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++namespace { ++ ++// ik_llama.cpp ships this table as a literal: it is iq1s_grid with 1 added to every int8 lane, so the ++// lanes hold 0, 1, 2 instead of -1, 0, 1. It is derived here from the iq1s_grid of ggml-common.h. ++// Filled on first use, not by a static initializer: dlopen runs those in every CPU variant library while ggml ++// scores them, so an AVX512-vectorized initializer would SIGILL on CPUs without AVX512. ++uint64_t iq1s_grid_us[NGRID_IQ1S]; ++bool iq1s_grid_us_init() { ++ static const bool ready = [] { ++ for (int i = 0; i < NGRID_IQ1S; ++i) { ++ uint64_t v = 0; ++ for (int k = 0; k < 8; ++k) { ++ v |= (uint64_t)(uint8_t)((int8_t)(iq1s_grid[i] >> 8*k) + 1) << 8*k; ++ } ++ iq1s_grid_us[i] = v; ++ } ++ return true; ++ }(); ++ return ready; ++} ++ ++} ++ ++#ifdef __x86_64__ ++ ++namespace { ++ ++template ++static inline void prefetch_row_ahead(const Block * x) { ++ _mm_prefetch((const char *)x + 2048, _MM_HINT_T2); ++} ++ ++template ++void mul_mat_iq1_s_q8_K(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ int ix0 = 0; ++ Q8 q8(info); ++ __m256i qx[8]; ++ __m256i scales[4]; ++ __m256 acc[nrc_y] = {}; ++ auto delta_mask = _mm_set1_epi16(-32768); // to avoid stupid overflow warnings when using 0x8000 ++ __m256i shuffle0 = _mm256_set_epi64x(0x0302030203020302, 0x0100010001000100, 0x0302030203020302, 0x0100010001000100); ++ for (int ix = ix0; ix < nrc_x; ++ix) { ++ auto iq1s = (const block_iq1_s *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < n/QK_K; ++ibl) { ++ prefetch_row_ahead(iq1s + ibl); ++ float d = GGML_FP16_TO_FP32(iq1s[ibl].d); ++ auto qhb = _mm_loadu_si128((const __m128i *)iq1s[ibl].qh); ++ auto scales128 = _mm_and_si128(_mm_srli_epi16(qhb, 12), _mm_set1_epi16(7)); ++ scales128 = _mm_add_epi16(_mm_slli_epi16(scales128, 1), _mm_set1_epi16(1)); ++#ifdef HAVE_FANCY_SIMD ++ auto mask = _mm_cmpeq_epi16_mask(_mm_and_si128(qhb, delta_mask), delta_mask); ++ auto deltas128 = _mm_mask_blend_epi16(mask, _mm_set1_epi16(-7), _mm_set1_epi16(-9)); ++#else ++ auto mask = _mm_cmpeq_epi16(_mm_and_si128(qhb, delta_mask), delta_mask); ++ auto deltas128 = _mm_or_si128(_mm_and_si128(mask, _mm_set1_epi16(-9)), _mm_andnot_si128(mask, _mm_set1_epi16(-7))); ++#endif ++ deltas128 = _mm_mullo_epi16(scales128, deltas128); ++ scales128 = _mm_slli_epi16(scales128, 3); ++ auto deltas_l = _mm_unpacklo_epi16(deltas128, deltas128); ++ auto deltas_h = _mm_unpackhi_epi16(deltas128, deltas128); ++ auto deltas = MM256_SET_M128I(deltas_h, deltas_l); // blocks 0,0, 1,1, 2,2, ..., 7,7 ++ auto all_scales = MM256_SET1_M128I(scales128); ++ auto shuffle = shuffle0; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ scales[ib64] = _mm256_shuffle_epi8(all_scales, shuffle); ++ shuffle = _mm256_add_epi8(shuffle, _mm256_set1_epi8(4)); ++ } ++ const uint8_t * qs = iq1s[ibl].qs; ++ const uint16_t * qh = iq1s[ibl].qh; ++ for (int ib = 0; ib < QK_K/32; ib += 2) { ++ qx[ib+0] = _mm256_set_epi64x(iq1s_grid_us[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid_us[qs[2] | ((qh[ib+0] << 2) & 0x700)], ++ iq1s_grid_us[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid_us[qs[0] | ((qh[ib+0] << 8) & 0x700)]); ++ qx[ib+1] = _mm256_set_epi64x(iq1s_grid_us[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid_us[qs[6] | ((qh[ib+1] << 2) & 0x700)], ++ iq1s_grid_us[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid_us[qs[4] | ((qh[ib+1] << 8) & 0x700)]); ++ qs += 8; ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto bsums = q8.load_bsums(iy, ibl); ++ auto sumi = _mm256_setzero_si256(); ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ auto qy1 = q8.load_quants(iy, ibl, 2*ib64+0); ++ auto qy2 = q8.load_quants(iy, ibl, 2*ib64+1); ++#ifdef HAVE_FANCY_SIMD ++ auto dot1 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), qx[2*ib64+0], qy1); ++ auto dot2 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), qx[2*ib64+1], qy2); ++ sumi = _mm256_dpwssd_epi32(sumi, scales[ib64], _mm256_packs_epi32(dot1, dot2)); ++#else ++ auto dot1 = _mm256_maddubs_epi16(qx[2*ib64+0], qy1); ++ auto dot2 = _mm256_maddubs_epi16(qx[2*ib64+1], qy2); ++ auto dot = _mm256_add_epi16(_mm256_unpacklo_epi64(dot1, dot2), _mm256_unpackhi_epi64(dot1, dot2)); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(scales[ib64], dot)); ++#endif ++ } ++#ifdef HAVE_FANCY_SIMD ++ sumi = _mm256_dpwssd_epi32(sumi, bsums, deltas); ++#else ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(bsums, deltas)); ++#endif ++ acc[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d*q8.scale(iy, ibl)), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, 0.125f*hsum_float_8(acc[iy])); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++ ++template ++void mul_mat_iq1_m_q8_K(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ int ix0 = 0; ++ Q8 q8(info); ++ __m256i qx[8]; ++ __m256 acc[nrc_y] = {}; ++ auto scale_shuffle = _mm256_set_epi64x(0x0706070607060706, 0x0504050405040504, 0x0302030203020302, 0x0100010001000100); ++ auto delta_mask = _mm256_set_epi64x(0x8000, 0x0800, 0x0080, 0x0008); ++ iq1m_scale_t scale; ++ union { __m256i vec; int16_t val[16]; } helper; ++ for (int ix = ix0; ix < nrc_x; ++ix) { ++ auto iq1m = (const block_iq1_m *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < n/QK_K; ++ibl) { ++ prefetch_row_ahead(iq1m + ibl); ++ const uint16_t * sc = (const uint16_t *)iq1m[ibl].scales; // 4 x uint16_t, each containing 4 scales ++ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); ++ float d = GGML_FP16_TO_FP32(scale.f16); ++ auto qs = iq1m[ibl].qs; ++ auto qh = iq1m[ibl].qh; ++ auto aux = _mm_loadl_epi64((const __m128i *)iq1m[ibl].scales); ++ auto sc16 = _mm256_shuffle_epi8(MM256_SET1_M128I(aux), scale_shuffle); ++ sc16 = _mm256_and_si256(sc16, _mm256_set1_epi64x(0x0e0001c000380007)); ++ sc16 = _mm256_mullo_epi16(sc16, _mm256_set1_epi64x(0x0001000800400200)); ++ helper.vec = _mm256_add_epi8(_mm256_srli_epi16(sc16, 8), _mm256_set1_epi16(1)); ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ qx[2*ib64+0] = _mm256_set_epi64x(iq1s_grid_us[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid_us[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)], ++ iq1s_grid_us[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid_us[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]); ++ qx[2*ib64+1] = _mm256_set_epi64x(iq1s_grid_us[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid_us[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)], ++ iq1s_grid_us[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid_us[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]); ++ //auto delta1 = _mm256_set_epi64x(qh[1] & 0x80 ? 0x0909090909090909 : 0x0707070707070707, ++ // qh[1] & 0x08 ? 0x0909090909090909 : 0x0707070707070707, ++ // qh[0] & 0x80 ? 0x0909090909090909 : 0x0707070707070707, ++ // qh[0] & 0x08 ? 0x0909090909090909 : 0x0707070707070707); ++ //auto delta2 = _mm256_set_epi64x(qh[3] & 0x80 ? 0x0909090909090909 : 0x0707070707070707, ++ // qh[3] & 0x08 ? 0x0909090909090909 : 0x0707070707070707, ++ // qh[2] & 0x80 ? 0x0909090909090909 : 0x0707070707070707, ++ // qh[2] & 0x08 ? 0x0909090909090909 : 0x0707070707070707); ++ auto qh16 = (const uint16_t *)qh; ++ auto delta1 = _mm256_cmpeq_epi64(_mm256_and_si256(_mm256_set1_epi64x(qh16[0]), delta_mask), delta_mask); ++ auto delta2 = _mm256_cmpeq_epi64(_mm256_and_si256(_mm256_set1_epi64x(qh16[1]), delta_mask), delta_mask); ++ delta1 = _mm256_sub_epi8(_mm256_set1_epi8(8), _mm256_or_si256(delta1, _mm256_set1_epi8(1))); ++ delta2 = _mm256_sub_epi8(_mm256_set1_epi8(8), _mm256_or_si256(delta2, _mm256_set1_epi8(1))); ++ qx[2*ib64+0] = _mm256_sub_epi8(_mm256_slli_epi16(qx[2*ib64+0], 3), delta1); ++ qx[2*ib64+1] = _mm256_sub_epi8(_mm256_slli_epi16(qx[2*ib64+1], 3), delta2); ++ qs += 8; ++ qh += 4; ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = _mm256_setzero_si256(); ++ for (int j = 0; j < 8; ++j) { ++ auto p = _mm256_maddubs_epi16(_mm256_sign_epi8(qx[j], qx[j]), _mm256_sign_epi8(q8.load_quants(iy, ibl, j), qx[j])); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(p, MM256_SET_M128I(_mm_set1_epi16(helper.val[2*j+1]), _mm_set1_epi16(helper.val[2*j+0])))); ++ } ++ acc[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d*q8.scale(iy, ibl)), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, 0.125f*hsum_float_8(acc[iy])); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++ ++} // namespace ++ ++#else ++// --------------------------------- __aarch64__ -------------------------------------- ++ ++namespace { ++ ++template ++void mul_mat_iq1_s_q8_K(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ Q8 q8(info); ++ int8x16_t qx[16]; ++ int32x4_t scales[2]; ++ int16x4_t deltas[2]; ++ float32x4_t acc[nrc_y] = {}; ++ auto delta_mask = vdupq_n_u16(0x8000); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ auto iq1s = (const block_iq1_s *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < n/QK_K; ++ibl) { ++ float d = GGML_FP16_TO_FP32(iq1s[ibl].d); ++ auto qhb = vld1q_u16(iq1s[ibl].qh); ++ auto scales128 = vandq_u16(vshrq_n_u16(qhb, 12), vdupq_n_u16(7)); ++ scales128 = vaddq_u16(vshlq_n_u16(scales128, 1), vdupq_n_u16(1)); ++ auto mask = vceqq_u16(vandq_u16(qhb, delta_mask), delta_mask); ++ // Note: we explicitly assume IQ1S_DELTA = 0.125 ++ auto deltas128 = vsubq_s16(vbicq_s16(scales128, mask), vandq_s16(scales128, mask)); ++ //auto deltas128 = vorrq_s16(vandq_s16(vdupq_n_s16(-1), mask), vbicq_s16(vdupq_n_s16(1), mask)); ++ //deltas128 = vmulq_s16(scales128, deltas128); ++ scales128 = vshlq_n_u16(scales128, 3); ++ auto qs = iq1s[ibl].qs; ++ auto qh = iq1s[ibl].qh; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ qx[4*ib64+0] = vreinterpretq_s8_u64(uint64x2_t{iq1s_grid[qs[0] | ((qh[2*ib64+0] << 8) & 0x700)], iq1s_grid[qs[1] | ((qh[2*ib64+0] << 5) & 0x700)]}); ++ qx[4*ib64+1] = vreinterpretq_s8_u64(uint64x2_t{iq1s_grid[qs[2] | ((qh[2*ib64+0] << 2) & 0x700)], iq1s_grid[qs[3] | ((qh[2*ib64+0] >> 1) & 0x700)]}); ++ qx[4*ib64+2] = vreinterpretq_s8_u64(uint64x2_t{iq1s_grid[qs[4] | ((qh[2*ib64+1] << 8) & 0x700)], iq1s_grid[qs[5] | ((qh[2*ib64+1] << 5) & 0x700)]}); ++ qx[4*ib64+3] = vreinterpretq_s8_u64(uint64x2_t{iq1s_grid[qs[6] | ((qh[2*ib64+1] << 2) & 0x700)], iq1s_grid[qs[7] | ((qh[2*ib64+1] >> 1) & 0x700)]}); ++ qs += 8; ++ } ++ scales[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16 (scales128))); ++ scales[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales128))); ++ deltas[0] = vget_low_s16 (deltas128); ++ deltas[1] = vget_high_s16(deltas128); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto bsums = q8.load_bsums8(iy, ibl); ++ auto sumi = vdupq_n_s32(0); ++ sumi = vmlal_s16(sumi, deltas[0], vget_low_s16 (bsums)); ++ sumi = vmlal_s16(sumi, deltas[1], vget_high_s16(bsums)); ++ for (int k = 0; k < QK_K/128; ++k) { ++ auto qy = q8.load_quants_64(iy, ibl, 2*k+0); ++ auto dot1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), qx[8*k+0], qy.val[0]), qx[8*k+1], qy.val[1]); ++ auto dot2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), qx[8*k+2], qy.val[2]), qx[8*k+3], qy.val[3]); ++ auto dot12 = vpaddq_s32(dot1, dot2); ++ qy = q8.load_quants_64(iy, ibl, 2*k+1); ++ auto dot3 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), qx[8*k+4], qy.val[0]), qx[8*k+5], qy.val[1]); ++ auto dot4 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), qx[8*k+6], qy.val[2]), qx[8*k+7], qy.val[3]); ++ auto dot34 = vpaddq_s32(dot3, dot4); ++ auto dot = vpaddq_s32(dot12, dot34); ++ sumi = vmlaq_s32(sumi, dot, scales[k]); ++ } ++ acc[iy] = vfmaq_f32(acc[iy], vdupq_n_f32(d*q8.scale(iy, ibl)), vcvtq_f32_s32(sumi)); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, 0.125f*vaddvq_f32(acc[iy])); ++ acc[iy] = vdupq_n_f32(0); ++ } ++ } ++} ++ ++template ++void mul_mat_iq1_m_q8_K(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ Q8 q8(info); ++ int8x16x2_t qx[8]; ++ int32x4x4_t scales; ++ float32x4_t acc[nrc_y] = {}; ++ uint8x16x2_t scale_shuffle = {vreinterpretq_u8_u64(uint64x2_t{0x0100010001000100, 0x0302030203020302}), ++ vreinterpretq_u8_u64(uint64x2_t{0x0504050405040504, 0x0706070607060706})}; ++ uint64x2x2_t delta_mask = {uint64x2_t{0x0008, 0x0080}, uint64x2_t{0x0800, 0x8000}}; ++ iq1m_scale_t block_scale; ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ auto iq1m = (const block_iq1_m *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < n/QK_K; ++ibl) { ++ const uint16_t * sc = (const uint16_t *)iq1m[ibl].scales; // 4 x uint16_t, each containing 4 scales ++ block_scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); ++ float d = GGML_FP16_TO_FP32(block_scale.f16); ++ auto qs = iq1m[ibl].qs; ++ auto qh = iq1m[ibl].qh; ++ auto aux8 = vld1_u8(iq1m[ibl].scales); ++ auto aux16 = vcombine_u8(aux8, aux8); ++ uint16x8x2_t sc16 = { vreinterpretq_u16_u8(vqtbl1q_u8(aux16, scale_shuffle.val[0])), vreinterpretq_u16_u8(vqtbl1q_u8(aux16, scale_shuffle.val[1])) }; ++ sc16.val[0] = vmulq_u16(vandq_u16(sc16.val[0], vdupq_n_u64(0x0e0001c000380007)), vdupq_n_u64(0x0001000800400200)); ++ sc16.val[1] = vmulq_u16(vandq_u16(sc16.val[1], vdupq_n_u64(0x0e0001c000380007)), vdupq_n_u64(0x0001000800400200)); ++ sc16.val[0] = vaddq_u16(vshrq_n_u16(sc16.val[0], 8), vdupq_n_u16(1)); ++ sc16.val[1] = vaddq_u16(vshrq_n_u16(sc16.val[1], 8), vdupq_n_u16(1)); ++ scales.val[0] = vmovl_s16(vget_low_s16 (sc16.val[0])); ++ scales.val[1] = vmovl_s16(vget_high_s16(sc16.val[0])); ++ scales.val[2] = vmovl_s16(vget_low_s16 (sc16.val[1])); ++ scales.val[3] = vmovl_s16(vget_high_s16(sc16.val[1])); ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ qx[2*ib64+0] = {vreinterpretq_s8_u64(uint64x2_t{iq1s_grid_us[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)], ++ iq1s_grid_us[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)]}), ++ vreinterpretq_s8_u64(uint64x2_t{iq1s_grid_us[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)], ++ iq1s_grid_us[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)]})}; ++ qx[2*ib64+1] = {vreinterpretq_s8_u64(uint64x2_t{iq1s_grid_us[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)], ++ iq1s_grid_us[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)]}), ++ vreinterpretq_s8_u64(uint64x2_t{iq1s_grid_us[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)], ++ iq1s_grid_us[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)]})}; ++ auto qh16 = (const uint16_t *)qh; ++ auto h1 = vdupq_n_u64(qh16[0]); ++ auto h2 = vdupq_n_u64(qh16[1]); ++ auto delta1 = vsubq_s8(vdupq_n_s8(8), vorrq_s8(vreinterpretq_s8_u64(vceqq_u64(vandq_u64(h1, delta_mask.val[0]), delta_mask.val[0])), vdupq_n_s8(1))); ++ auto delta2 = vsubq_s8(vdupq_n_s8(8), vorrq_s8(vreinterpretq_s8_u64(vceqq_u64(vandq_u64(h1, delta_mask.val[1]), delta_mask.val[1])), vdupq_n_s8(1))); ++ qx[2*ib64+0].val[0] = vsubq_s8(vshlq_n_s8(qx[2*ib64+0].val[0], 3), delta1); ++ qx[2*ib64+0].val[1] = vsubq_s8(vshlq_n_s8(qx[2*ib64+0].val[1], 3), delta2); ++ delta1 = vsubq_s8(vdupq_n_s8(8), vorrq_s8(vreinterpretq_s8_u64(vceqq_u64(vandq_u64(h2, delta_mask.val[0]), delta_mask.val[0])), vdupq_n_s8(1))); ++ delta2 = vsubq_s8(vdupq_n_s8(8), vorrq_s8(vreinterpretq_s8_u64(vceqq_u64(vandq_u64(h2, delta_mask.val[1]), delta_mask.val[1])), vdupq_n_s8(1))); ++ qx[2*ib64+1].val[0] = vsubq_s8(vshlq_n_s8(qx[2*ib64+1].val[0], 3), delta1); ++ qx[2*ib64+1].val[1] = vsubq_s8(vshlq_n_s8(qx[2*ib64+1].val[1], 3), delta2); ++ qs += 8; ++ qh += 4; ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = vdupq_n_s32(0); ++ for (int j = 0; j < 4; ++j) { ++ auto y1 = q8.load_quants(iy, ibl, 2*j+0); ++ auto dot1 = ggml_vdotq_s32(vdupq_n_s32(0), qx[2*j+0].val[0], y1.val[0]); ++ auto dot2 = ggml_vdotq_s32(vdupq_n_s32(0), qx[2*j+0].val[1], y1.val[1]); ++ auto y2 = q8.load_quants(iy, ibl, 2*j+1); ++ auto dot3 = ggml_vdotq_s32(vdupq_n_s32(0), qx[2*j+1].val[0], y2.val[0]); ++ auto dot4 = ggml_vdotq_s32(vdupq_n_s32(0), qx[2*j+1].val[1], y2.val[1]); ++ auto dot = vpaddq_s32(vpaddq_s32(dot1, dot2), vpaddq_s32(dot3, dot4)); ++ sumi = vmlaq_s32(sumi, dot, scales.val[j]); ++ } ++ acc[iy] = vfmaq_f32(acc[iy], vdupq_n_f32(d*q8.scale(iy, ibl)), vcvtq_f32_s32(sumi)); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, 0.125f*vaddvq_f32(acc[iy])); ++ acc[iy] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++} // namespace ++ ++#endif ++ ++bool iqk_set_kernels_1bit(int typeA, mul_mat_t * funcs) { ++ switch (typeA) { ++ case GGML_TYPE_IQ1_S: IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_iq1_s_q8_K, funcs); break; ++ case GGML_TYPE_IQ1_M: IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_iq1_m_q8_K, funcs); break; ++ default: return false; ++ } ++ return iq1s_grid_us_init(); ++} ++ ++#endif // IQK_IMPLEMENT +diff --git a/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp b/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp +new file mode 100644 +index 00000000..72027c76 +--- /dev/null ++++ b/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp +@@ -0,0 +1,1169 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++// Adapted from ik_llama.cpp (https://github.com/ikawrakow/ik_llama.cpp) commit 009dce19, ++// ggml/src/iqk/{iqk_config.h, iqk_common.h, iqk_gemm_kquants.cpp, iqk_gemm_iquants.cpp, iqk_gemm_1bit.cpp, iqk_mul_mat.cpp}. ++// Only the kernels that read the upstream ggml block layouts with upstream Q8_K activations are kept. ++// ++// Permission is hereby granted, free of charge, to any person obtaining a copy ++// of this software and associated documentation files (the "Software"), to deal ++// in the Software without restriction, including without limitation the rights ++// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell ++// copies of the Software, and to permit persons to whom the Software is ++// furnished to do so, subject to the following conditions: ++// ++// The above copyright notice and this permission notice shall be included in all ++// copies or substantial portions of the Software. ++// ++// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ++// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++// SOFTWARE. ++// ++// IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS and IQ3_S times Q8_K (from iqk_gemm_iquants.cpp). ++ ++#include "iqk_common.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++#ifdef __x86_64__ ++ ++namespace { ++ ++inline __m256i get_scale_shuffle_8(int i) { ++ return _mm256_set1_epi16((2*i) | ((2*i+1) << 8)); ++} ++ ++inline void set_scales_8(const __m256i& all_scales, int j, __m256i * scales) { ++ scales[0] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+0)); ++ scales[1] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+1)); ++ scales[2] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+2)); ++ scales[3] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+3)); ++} ++ ++inline __m256i get_scale_shuffle_16(int i) { ++ static const uint8_t k_shuffle[128] = { ++ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, ++ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, ++ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11, ++ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15, ++ }; ++ return _mm256_loadu_si256((const __m256i*)k_shuffle + i); ++} ++ ++inline void set_scales_16(const __m256i& all_scales, __m256i * scales) { ++ scales[0] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(0)); ++ scales[1] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(1)); ++ scales[2] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(2)); ++ scales[3] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(3)); ++} ++ ++// TODO: find the bug that causes this to be called without HAVE_FANCY_SIMD, which triggers ++// writing 4 vvalues into scales, which is of size 2. ++inline void set_scales_8_iq(int j, const __m256i& all_scales, __m256i * scales) { ++//#ifdef HAVE_FANCY_SIMD ++ auto shuffle = j == 0 ? _mm256_set_epi64x(0x0302030203020302, 0x0100010001000100, 0x0302030203020302, 0x0100010001000100) ++ : _mm256_set_epi64x(0x0b0a0b0a0b0a0b0a, 0x0908090809080908, 0x0b0a0b0a0b0a0b0a, 0x0908090809080908); ++ scales[0] = _mm256_shuffle_epi8(all_scales, shuffle); ++ scales[1] = _mm256_shuffle_epi8(all_scales, _mm256_add_epi8(shuffle, _mm256_set1_epi8(4))); ++//#else ++// set_scales_8(all_scales, j, scales); ++//#endif ++} ++ ++inline void set_scales_16_iq(const __m256i& all_scales, __m256i * scales) { ++#ifdef HAVE_FANCY_SIMD ++ auto shuffle = _mm256_set_epi64x(0x0706070607060706, 0x0302030203020302, 0x0504050405040504, 0x0100010001000100); ++ scales[0] = _mm256_shuffle_epi8(all_scales, shuffle); ++ scales[1] = _mm256_shuffle_epi8(all_scales, _mm256_add_epi8(shuffle, _mm256_set1_epi8(8))); ++#else ++ set_scales_16(all_scales, scales); ++#endif ++} ++ ++struct SimpleBits { ++ __m256i values[4]; ++}; ++ ++struct EvenSignHelper { ++#if defined HAVE_FANCY_SIMD && defined __AVX512VPOPCNTDQ__ ++ union sbits_t { ++ __m128i vec; ++ __mmask32 mask[4]; ++ }; ++ IQK_ALWAYS_INLINE void sign_2_values(__m256i aux, __m256i * values) const { ++ aux = _mm256_and_si256(_mm256_srlv_epi32(aux, shifts), mask); ++ auto pcnt = _mm256_popcnt_epi32(aux); ++ sbits_t sbits; ++ sbits.vec = _mm256_cvtepi32_epi8(_mm256_or_si256(aux, _mm256_slli_epi32(_mm256_and_si256(pcnt, mone), 7))); ++ values[0] = _mm256_mask_sub_epi8(values[0], sbits.mask[0], _mm256_setzero_si256(), values[0]); ++ values[1] = _mm256_mask_sub_epi8(values[1], sbits.mask[1], _mm256_setzero_si256(), values[1]); ++ //auto sign_bits = _mm256_cvtepi32_epi8(_mm256_or_si256(aux, _mm256_slli_epi32(_mm256_and_si256(pcnt, mone), 7))); ++ //const __mmask32 * m32 = (const __mmask32 *)&sign_bits; ++ //values[0] = _mm256_mask_sub_epi8(values[0], m32[0], _mm256_setzero_si256(), values[0]); ++ //values[1] = _mm256_mask_sub_epi8(values[1], m32[1], _mm256_setzero_si256(), values[1]); ++ } ++ const __m256i shifts = _mm256_set_epi32(21, 14, 7, 0, 21, 14, 7, 0); ++ const __m256i mask = _mm256_set1_epi32(127); ++ const __m256i mone = _mm256_set1_epi32(1); ++#endif ++ inline void sign_value(uint32_t aux32, __m256i& value) const { ++ auto signs = _mm256_set_epi64x(keven_signs[(aux32 >> 21) & 127], keven_signs[(aux32 >> 14) & 127], ++ keven_signs[(aux32 >> 7) & 127], keven_signs[(aux32 >> 0) & 127]); ++ value = _mm256_sign_epi8(value, signs); ++ } ++}; ++ ++struct SignHelper { ++ inline __m256i make_signs(uint32_t sign_bits) const { ++ auto aux256 = _mm256_set1_epi32(sign_bits); ++ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256, mask1), mask2); ++ return _mm256_or_si256(_mm256_cmpeq_epi8(aux256, mask2), mone); ++ } ++// inline __m256i make_signs(const uint16_t * sign_bits) const { ++//#ifdef HAVE_FANCY_SIMD ++//#else ++// return make_signs(sign_bits[0] | (sign_bits[1] << 16)); ++//#endif ++// } ++ inline __m256i sign_value(const uint16_t * sign_bits, const __m256i& value) const { ++#ifdef HAVE_FANCY_SIMD ++ const __mmask32 * mask = (const __mmask32 *)sign_bits; ++ return _mm256_mask_sub_epi8(value, mask[0], _mm256_setzero_si256(), value); ++#else ++ return _mm256_sign_epi8(value, make_signs(sign_bits[0] | (sign_bits[1] << 16))); ++#endif ++ } ++ IQK_ALWAYS_INLINE void sign_4_values(const uint16_t * sign_bits, __m256i * values) const { ++ // Somehow the FANCY_SIMD version has become 50% slower for TG??? ++#ifdef z_HAVE_FANCY_SIMD ++ //__mmask32 mask[4]; std::memcpy(mask, sign_bits, 4*sizeof(__mmask32)); ++ const __mmask32 * mask = (const __mmask32 *)sign_bits; ++ values[0] = _mm256_mask_sub_epi8(values[0], mask[0], _mm256_setzero_si256(), values[0]); ++ values[1] = _mm256_mask_sub_epi8(values[1], mask[1], _mm256_setzero_si256(), values[1]); ++ values[2] = _mm256_mask_sub_epi8(values[2], mask[2], _mm256_setzero_si256(), values[2]); ++ values[3] = _mm256_mask_sub_epi8(values[3], mask[3], _mm256_setzero_si256(), values[3]); ++#else ++ auto s128 = _mm_loadu_si128((const __m128i *)sign_bits); ++ auto s256 = MM256_SET1_M128I(s128); ++ __m256i aux256; ++ auto shuffle = mask1; ++ auto step = _mm256_set1_epi8(4); ++ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(s256, shuffle), mask2); shuffle = _mm256_add_epi8(shuffle, step); ++ values[0] = _mm256_sign_epi8(values[0], _mm256_or_si256(_mm256_cmpeq_epi8(aux256, mask2), mone)); ++ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(s256, shuffle), mask2); shuffle = _mm256_add_epi8(shuffle, step); ++ values[1] = _mm256_sign_epi8(values[1], _mm256_or_si256(_mm256_cmpeq_epi8(aux256, mask2), mone)); ++ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(s256, shuffle), mask2); shuffle = _mm256_add_epi8(shuffle, step); ++ values[2] = _mm256_sign_epi8(values[2], _mm256_or_si256(_mm256_cmpeq_epi8(aux256, mask2), mone)); ++ aux256 = _mm256_and_si256(_mm256_shuffle_epi8(s256, shuffle), mask2); shuffle = _mm256_add_epi8(shuffle, step); ++ values[3] = _mm256_sign_epi8(values[3], _mm256_or_si256(_mm256_cmpeq_epi8(aux256, mask2), mone)); ++#endif ++ } ++ const __m256i mask1 = _mm256_set_epi64x(0x0303030303030303, 0x0202020202020202, 0x0101010101010101, 0x0000000000000000); ++ const __m256i mask2 = _mm256_set1_epi64x(0x8040201008040201ull); ++ const __m256i mone = _mm256_set1_epi8(1); ++}; ++ ++struct DequantizerIQ2XXS final : public BaseDequantizer { ++ DequantizerIQ2XXS(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ constexpr static int num_blocks = 8; ++ ++ union Data { ++ __m256i vec; ++ uint32_t val[8]; ++ }; ++ ++ inline __m128i load_scales(int i) { ++ d = 0.125f * GGML_FP16_TO_FP32(x[i].d); ++ const uint16_t * a16 = (const uint16_t *)x[i].qs; ++ auto scales = _mm_srli_epi16(_mm_set_epi16(a16[31], a16[27], a16[23], a16[19], a16[15], a16[11], a16[7], a16[3]), 12); ++ return _mm_or_si128(_mm_slli_epi16(scales, 1), _mm_set1_epi16(1)); ++ } ++ ++ inline void new_block(int i, __m256i * scales) { ++ auto sc16 = load_scales(i); ++ scales[0] = MM256_SET1_M128I(sc16); ++ } ++ inline void new_block_f(int i, __m256 * scales) { ++ auto sc16 = load_scales(i); ++ auto scf = _mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(sc16))); ++ auto scf_l = _mm256_castps256_ps128(scf); ++ auto scf_h = _mm256_extractf128_ps(scf, 1); ++ scales[0] = _mm256_set_m128(scf_l, scf_l); ++ scales[1] = _mm256_set_m128(scf_h, scf_h); ++ scales[2] = _mm256_mul_ps(scf, _mm256_set1_ps(-minv)); ++ } ++ ++ inline float new_block(int i, __m256i * scales, __m256i& mins) { ++ auto sc16 = load_scales(i); ++ mins = scb.shuffle(sc16); ++ scales[0] = MM256_SET1_M128I(sc16); ++ return -d*minv; ++ } ++ ++ inline static void make4(const uint32_t * aux32, __m256i * values) { ++ const uint8_t * aux8 = (const uint8_t *)aux32; ++ values[0] = _mm256_set_epi64x(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[ 1]], iq2xxs_grid[aux8[ 0]]); ++ values[1] = _mm256_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[ 9]], iq2xxs_grid[aux8[ 8]]); ++ values[2] = _mm256_set_epi64x(iq2xxs_grid[aux8[19]], iq2xxs_grid[aux8[18]], iq2xxs_grid[aux8[17]], iq2xxs_grid[aux8[16]]); ++ values[3] = _mm256_set_epi64x(iq2xxs_grid[aux8[27]], iq2xxs_grid[aux8[26]], iq2xxs_grid[aux8[25]], iq2xxs_grid[aux8[24]]); ++ } ++ ++ IQK_ALWAYS_INLINE void sign_values(const uint32_t * aux32, __m256i * values) const { ++#if defined z_HAVE_FANCY_SIMD && defined __AVX512VPOPCNTDQ__ ++ esh.sign_2_values(MM256_SET_M128I(_mm_set1_epi32(aux32[3]), _mm_set1_epi32(aux32[1])), values+0); ++ esh.sign_2_values(MM256_SET_M128I(_mm_set1_epi32(aux32[7]), _mm_set1_epi32(aux32[5])), values+2); ++#else ++ esh.sign_value(aux32[1], values[0]); ++ esh.sign_value(aux32[3], values[1]); ++ esh.sign_value(aux32[5], values[2]); ++ esh.sign_value(aux32[7], values[3]); ++#endif ++ } ++ inline void make4_signed(const uint32_t * aux32, const __m256i& min_value, __m256i * values) const { ++ make4(aux32, values); ++ sign_values(aux32, values); ++ for (int k = 0; k < 4; ++k) values[k] = _mm256_add_epi8(values[k], min_value); ++ } ++ inline void make4(const uint32_t * aux32, __m256i * values, __m256i * q8) const { ++ make4(aux32, values); ++ sign_values(aux32, q8); ++ } ++ inline void prepare(int i, int j) { ++ Data data; data.vec = _mm256_loadu_si256((const __m256i *)x[i].qs + j); ++ make4_signed(data.val, min_value, bits.values); ++ } ++ inline void prepare(int i, int j, const Q8<1>& q8, __m256i * q8_quants) { ++ for (int k = 0; k < 4; ++k) q8_quants[k] = q8.load_quants(0, i, 4*j+k); ++ Data data; data.vec = _mm256_loadu_si256((const __m256i *)x[i].qs + j); ++ make4(data.val, bits.values, q8_quants); ++ } ++ inline void prepare(int i, int j, __m256i * q8_quants) { ++ Data data; data.vec = _mm256_loadu_si256((const __m256i *)x[i].qs + j); ++ make4(data.val, bits.values, q8_quants); ++ } ++ ++ constexpr static int minv = 43; ++ SimpleBits bits; ++ Scales8KBase scb; ++ EvenSignHelper esh; ++ const __m256i min_value = _mm256_set1_epi8(minv); ++ const __m256i shuffle = _mm256_set_epi32(7, 5, 3, 1, 7, 5, 3, 1); ++}; ++ ++struct DequantizerIQ2XS final : public BaseDequantizer { ++ DequantizerIQ2XS(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ constexpr static int num_blocks = 16; ++ ++ static inline __m256i make_scales(const uint8_t * scales) { ++ auto tmp = _mm_loadl_epi64((const __m128i *)scales); ++ auto all = _mm_and_si128(_mm_unpacklo_epi8(tmp, _mm_srli_epi16(tmp, 4)), _mm_set1_epi8(0xf)); ++ auto scales8 = _mm_or_si128(_mm_slli_epi16(all, 1), _mm_set1_epi8(1)); ++ return _mm256_cvtepi8_epi16(scales8); ++ } ++ ++ inline __m256i load_scales(int i) { ++ d = 0.125f * GGML_FP16_TO_FP32(x[i].d); ++ return make_scales(x[i].scales); ++ } ++ inline static void prepare_scales(const __m256i& all, __m256i * scales) { ++ auto scales_l = _mm256_castsi256_si128(all); ++ auto scales_h = _mm256_extractf128_si256(all, 1); ++ scales[0] = MM256_SET1_M128I(scales_l); ++ scales[1] = MM256_SET1_M128I(scales_h); ++ } ++ ++ inline void new_block(int i, __m256i * scales) { ++ prepare_scales(load_scales(i), scales); ++ } ++ inline float new_block(int i, __m256i * scales, __m256i& mins) { ++ mins = load_scales(i); ++ prepare_scales(mins, scales); ++ return -d*minv; ++ } ++ ++ struct Helper { ++ const __m256i mone = _mm256_set1_epi8(1); ++ const __m256i mask = _mm256_set1_epi64x(0x8040201008040201); ++ //const __m256i bhelper = _mm256_set_epi64x(0x8000008000808000, 0x0080800080000080, 0x8000008000808000, 0x0080800080000080); ++ const __m256i bhelper = load_bhelper(); ++ const __m256i shuff1 = _mm256_set_epi64x(0x0606060606060606, 0x0404040404040404, 0x0202020202020202, 0x0000000000000000); ++ const __m256i shuff2 = _mm256_set_epi64x(0x0e0e0e0e0e0e0e0e, 0x0c0c0c0c0c0c0c0c, 0x0a0a0a0a0a0a0a0a, 0x0808080808080808); ++ static __m256i load_bhelper() { ++ static const uint8_t k_bit_helper[32] = { ++ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, ++ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, ++ }; ++ return _mm256_loadu_si256((const __m256i*)k_bit_helper); ++ } ++ }; ++ ++ union index_t { ++ __m256i vec; ++ uint16_t val[8]; ++ }; ++ ++ inline static void make4(const __m256i& data, const __m256i& mask, __m256i * values) { ++ index_t idx; ++ idx.vec = _mm256_and_si256(data, mask); ++ values[0] = _mm256_set_epi64x(iq2xs_grid[idx.val[ 3]], iq2xs_grid[idx.val[ 2]], iq2xs_grid[idx.val[ 1]], iq2xs_grid[idx.val[ 0]]); ++ values[1] = _mm256_set_epi64x(iq2xs_grid[idx.val[ 7]], iq2xs_grid[idx.val[ 6]], iq2xs_grid[idx.val[ 5]], iq2xs_grid[idx.val[ 4]]); ++ values[2] = _mm256_set_epi64x(iq2xs_grid[idx.val[11]], iq2xs_grid[idx.val[10]], iq2xs_grid[idx.val[ 9]], iq2xs_grid[idx.val[ 8]]); ++ values[3] = _mm256_set_epi64x(iq2xs_grid[idx.val[15]], iq2xs_grid[idx.val[14]], iq2xs_grid[idx.val[13]], iq2xs_grid[idx.val[12]]); ++ } ++ inline static void sign_value(const __m256i& sign_bits, const __m256i& shuffle, const __m256i& mask, ++ const __m256i& mone, __m256i& value) { ++ auto signs = _mm256_shuffle_epi8(sign_bits, shuffle); ++ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, mask), mask); ++ value = _mm256_sign_epi8(value, _mm256_or_si256(signs, mone)); ++ } ++#if defined HAVE_FANCY_SIMD && defined __AVX512VPOPCNTDQ__ ++ static IQK_ALWAYS_INLINE inline void sign_values_popcnt(const __m256i& data, __m256i * values) { ++ auto partial_bits = _mm256_cvtepi16_epi8(_mm256_srli_epi16(data, 9)); ++ auto pcnt = _mm_popcnt_epi8(partial_bits); ++ auto full_bits = _mm_or_si128(partial_bits, _mm_slli_epi16(_mm_and_si128(pcnt, _mm_set1_epi8(1)), 7)); ++ const __mmask32 * m32 = (const __mmask32 *)&full_bits; ++ auto zero = _mm256_setzero_si256(); ++ values[0] = _mm256_mask_sub_epi8(values[0], m32[0], zero, values[0]); ++ values[1] = _mm256_mask_sub_epi8(values[1], m32[1], zero, values[1]); ++ values[2] = _mm256_mask_sub_epi8(values[2], m32[2], zero, values[2]); ++ values[3] = _mm256_mask_sub_epi8(values[3], m32[3], zero, values[3]); ++ } ++#else ++ static IQK_ALWAYS_INLINE inline void sign_values_helper(const __m256i& data, const Helper& helper, __m256i * values) { ++ auto psb1 = _mm256_srli_epi16(data, 9); ++ auto psb2 = _mm256_srli_epi16(data, 13); ++ auto psbc = _mm256_xor_si256(psb1, psb2); ++ auto oddb = _mm256_shuffle_epi8(helper.bhelper, psbc); ++ auto full = _mm256_or_si256(psb1, oddb); ++ auto full_l = _mm256_castsi256_si128(full); ++ auto full_h = _mm256_extractf128_si256(full, 1); ++ auto full_1 = MM256_SET1_M128I(full_l); ++ auto full_2 = MM256_SET1_M128I(full_h); ++ sign_value(full_1, helper.shuff1, helper.mask, helper.mone, values[0]); ++ sign_value(full_1, helper.shuff2, helper.mask, helper.mone, values[1]); ++ sign_value(full_2, helper.shuff1, helper.mask, helper.mone, values[2]); ++ sign_value(full_2, helper.shuff2, helper.mask, helper.mone, values[3]); ++ } ++#endif ++ IQK_ALWAYS_INLINE inline void sign_values(const __m256i& data, __m256i * values) const { ++#if defined HAVE_FANCY_SIMD && defined __AVX512VPOPCNTDQ__ ++ sign_values_popcnt(data, values); ++#else ++ sign_values_helper(data, helper, values); ++#endif ++ } ++ inline void make4_signed(const uint16_t * qs, const __m256i& m511, ++ const __m256i& min_value, __m256i * values) const { ++ auto q2 = _mm256_loadu_si256((const __m256i *)qs); ++ make4(q2, m511, values); ++ sign_values(q2, values); ++ for (int k = 0; k < 4; ++k) values[k] = _mm256_add_epi8(values[k], min_value); ++ } ++ inline void make4(const uint16_t * qs, const __m256i& m511, __m256i * values, __m256i * q8) const { ++ auto q2 = _mm256_loadu_si256((const __m256i *)qs); ++ make4(q2, m511, values); ++ sign_values(q2, q8); ++ } ++ ++ inline void prepare(int i, int j) { ++ make4_signed(x[i].qs + 16*j, idx_mask, min_value, bits.values); ++ } ++ inline void prepare(int i, int j, const Q8<1>& q8, __m256i * q8_quants) { ++ for (int k = 0; k < 4; ++k) q8_quants[k] = q8.load_quants(0, i, 4*j+k); ++ make4(x[i].qs + 16*j, idx_mask, bits.values, q8_quants); ++ } ++ inline void prepare_signed(int i, int j, __m256i * us) { ++ auto q2 = _mm256_loadu_si256((const __m256i *)x[i].qs+j); ++ make4(q2, idx_mask, us); ++ for (int k = 0; k < 4; ++k) bits.values[k] = us[k]; ++ sign_values(q2, bits.values); ++ } ++ IQK_ALWAYS_INLINE inline void prepare_signed(int i, int j, __m256i * us, __m256i * s) { ++ auto q2 = _mm256_loadu_si256((const __m256i *)x[i].qs+j); ++ make4(q2, idx_mask, us); ++ sign_values(q2, s); ++ } ++ ++ constexpr static int minv = 43; ++ ++ SimpleBits bits; ++#if !(defined HAVE_FANCY_SIMD && defined __AVX512VPOPCNTDQ__) ++ Helper helper; ++#endif ++ const __m256i idx_mask = _mm256_set1_epi16(511); ++ const __m256i min_value = _mm256_set1_epi8(minv); ++ ++}; ++ ++struct DequantizerIQ2S final : public BaseDequantizer { ++ DequantizerIQ2S(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ constexpr static int num_blocks = 16; ++ ++ static inline __m256i make_scales(const uint8_t * scales) { ++ auto tmp = _mm_loadl_epi64((const __m128i *)scales); ++ auto all = _mm_and_si128(_mm_unpacklo_epi8(tmp, _mm_srli_epi16(tmp, 4)), _mm_set1_epi8(0xf)); ++ auto scales8 = _mm_or_si128(_mm_slli_epi16(all, 1), _mm_set1_epi8(1)); ++ return _mm256_cvtepi8_epi16(scales8); ++ } ++ inline __m256i load_scales(int i) { ++ d = 0.125f * GGML_FP16_TO_FP32(x[i].d); ++ return make_scales(x[i].scales); ++ } ++ inline static void prepare_scales(const __m256i& all, __m256i * scales) { ++ auto scales_l = _mm256_castsi256_si128(all); ++ auto scales_h = _mm256_extractf128_si256(all, 1); ++ scales[0] = MM256_SET1_M128I(scales_l); ++ scales[1] = MM256_SET1_M128I(scales_h); ++ } ++ ++ inline void new_block(int i, __m256i * scales) { ++ prepare_scales(load_scales(i), scales); ++ } ++ inline float new_block(int i, __m256i * scales, __m256i& mins) { ++ mins = load_scales(i); ++ prepare_scales(mins, scales); ++ return -d*minv; ++ } ++ ++ union index_t { ++ __m256i vec; ++ uint32_t val[8]; ++ }; ++ ++ inline static void make2(const uint8_t * qs, const uint8_t * qh, const __m256i& idx_shift, const __m256i& idx_mask, __m256i * values) { ++ auto idx_l = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)qs)); ++ auto idx_h = MM256_SET_M128I(_mm_set1_epi32(qh[1]), _mm_set1_epi32(qh[0])); ++ index_t idx; ++ idx.vec = _mm256_or_si256(idx_l, _mm256_and_si256(_mm256_sllv_epi32(idx_h, idx_shift), idx_mask)); ++ values[0] = _mm256_set_epi64x(iq2s_grid[idx.val[3]], iq2s_grid[idx.val[2]], iq2s_grid[idx.val[1]], iq2s_grid[idx.val[0]]); ++ values[1] = _mm256_set_epi64x(iq2s_grid[idx.val[7]], iq2s_grid[idx.val[6]], iq2s_grid[idx.val[5]], iq2s_grid[idx.val[4]]); ++ } ++ inline static void make2_signed(const SignHelper& sh, const uint8_t * qs, const uint8_t * qh, const uint16_t * sidx, ++ const __m256i& idx_shift, const __m256i& idx_mask, const __m256i& min_value, __m256i * values) { ++ make2(qs, qh, idx_shift, idx_mask, values); ++ values[0] = _mm256_add_epi8(sh.sign_value(sidx+0, values[0]), min_value); ++ values[1] = _mm256_add_epi8(sh.sign_value(sidx+2, values[1]), min_value); ++ } ++ ++ inline void prepare(int i, int j) { ++ auto qs = x[i].qs + 16*j; ++ auto qh = x[i].qh + 4*j; ++ const uint16_t * signs = (const uint16_t *)(x[i].qs + QK_K/8) + 8*j; ++ make2_signed(sh, qs+0, qh+0, signs+0, idx_shift, idx_mask, min_value, bits.values+0); ++ make2_signed(sh, qs+8, qh+2, signs+4, idx_shift, idx_mask, min_value, bits.values+2); ++ } ++ inline void prepare(int i, int j, const Q8<1>& q8, __m256i * q8_quants) { ++ auto qs = x[i].qs + 16*j; ++ auto qh = x[i].qh + 4*j; ++ const uint16_t * signs = (const uint16_t *)(x[i].qs + QK_K/8) + 8*j; ++ make2(qs+0, qh+0, idx_shift, idx_mask, bits.values+0); ++ make2(qs+8, qh+2, idx_shift, idx_mask, bits.values+2); ++ q8_quants[0] = _mm256_sign_epi8(q8.load_quants(0, i, 4*j+0), sh.make_signs(signs[0] | (signs[1] << 16))); ++ q8_quants[1] = _mm256_sign_epi8(q8.load_quants(0, i, 4*j+1), sh.make_signs(signs[2] | (signs[3] << 16))); ++ q8_quants[2] = _mm256_sign_epi8(q8.load_quants(0, i, 4*j+2), sh.make_signs(signs[4] | (signs[5] << 16))); ++ q8_quants[3] = _mm256_sign_epi8(q8.load_quants(0, i, 4*j+3), sh.make_signs(signs[6] | (signs[7] << 16))); ++ } ++ static inline void prepare(const uint8_t * qs, const uint8_t * qh, const uint16_t * signs, const SignHelper& sh, __m256i * values) { ++ auto idx_shift = _mm256_set_epi32(2, 4, 6, 8, 2, 4, 6, 8); ++ auto idx_mask = _mm256_set1_epi32(0x300); ++ make2(qs+0, qh+0, idx_shift, idx_mask, values+0); ++ make2(qs+8, qh+2, idx_shift, idx_mask, values+2); ++ values[0] = _mm256_sign_epi8(values[0], sh.make_signs(signs[0] | (signs[1] << 16))); ++ values[1] = _mm256_sign_epi8(values[1], sh.make_signs(signs[2] | (signs[3] << 16))); ++ values[2] = _mm256_sign_epi8(values[2], sh.make_signs(signs[4] | (signs[5] << 16))); ++ values[3] = _mm256_sign_epi8(values[3], sh.make_signs(signs[6] | (signs[7] << 16))); ++ } ++ inline void prepare_signed(int i, int j, __m256i * us, __m256i * s) { ++ auto qs = x[i].qs + 16*j; ++ auto qh = x[i].qh + 4*j; ++ const uint16_t * signs = (const uint16_t *)(x[i].qs + QK_K/8) + 8*j; ++ make2(qs+0, qh+0, idx_shift, idx_mask, us+0); ++ make2(qs+8, qh+2, idx_shift, idx_mask, us+2); ++ s[0] = _mm256_sign_epi8(s[0], sh.make_signs(signs[0] | (signs[1] << 16))); ++ s[1] = _mm256_sign_epi8(s[1], sh.make_signs(signs[2] | (signs[3] << 16))); ++ s[2] = _mm256_sign_epi8(s[2], sh.make_signs(signs[4] | (signs[5] << 16))); ++ s[3] = _mm256_sign_epi8(s[3], sh.make_signs(signs[6] | (signs[7] << 16))); ++ } ++ inline void prepare_signed(int i, int j, __m256i * us) { ++ auto qs = x[i].qs + 16*j; ++ auto qh = x[i].qh + 4*j; ++ const uint16_t * signs = (const uint16_t *)(x[i].qs + QK_K/8) + 8*j; ++ make2(qs+0, qh+0, idx_shift, idx_mask, us+0); ++ make2(qs+8, qh+2, idx_shift, idx_mask, us+2); ++ bits.values[0] = _mm256_sign_epi8(us[0], sh.make_signs(signs[0] | (signs[1] << 16))); ++ bits.values[1] = _mm256_sign_epi8(us[1], sh.make_signs(signs[2] | (signs[3] << 16))); ++ bits.values[2] = _mm256_sign_epi8(us[2], sh.make_signs(signs[4] | (signs[5] << 16))); ++ bits.values[3] = _mm256_sign_epi8(us[3], sh.make_signs(signs[6] | (signs[7] << 16))); ++ } ++ ++ constexpr static int minv = 43; ++ ++ SimpleBits bits; ++ SignHelper sh; ++ const __m256i idx_shift = _mm256_set_epi32(2, 4, 6, 8, 2, 4, 6, 8); ++ const __m256i idx_mask = _mm256_set1_epi32(0x300); ++ const __m256i min_value = _mm256_set1_epi8(minv); ++ ++}; ++ ++struct DequantizerIQ3XXS final : public BaseDequantizer { ++ DequantizerIQ3XXS(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ constexpr static int num_blocks = 8; ++ ++ inline __m128i prepare_scales(int i) { ++ d = 0.25f * GGML_FP16_TO_FP32(x[i].d); ++ auto tmp = _mm256_loadu_si256((const __m256i *)(x[i].qs + QK_K/4)); ++ auto scales32 = _mm256_srli_epi32(tmp, 28); ++ scales32 = _mm256_or_si256(_mm256_slli_epi32(scales32, 1), _mm256_set1_epi32(1)); ++ return _mm_packs_epi32(_mm256_castsi256_si128(scales32), _mm256_extractf128_si256(scales32, 1)); ++ } ++ ++ inline void new_block(int i, __m256i * scales) { ++ auto scales16 = prepare_scales(i); ++ scales[0] = MM256_SET1_M128I(scales16); ++ } ++ inline void new_block_f(int i, __m256 * scales) { ++ auto sc16 = prepare_scales(i); ++ auto scf = _mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(sc16))); ++ auto scf_l = _mm256_castps256_ps128(scf); ++ auto scf_h = _mm256_extractf128_ps(scf, 1); ++ scales[0] = _mm256_set_m128(scf_l, scf_l); ++ scales[1] = _mm256_set_m128(scf_h, scf_h); ++ scales[2] = _mm256_mul_ps(scf, _mm256_set1_ps(-minv)); ++ } ++ inline float new_block(int i, __m256i * scales, __m256i& mins) { ++ auto scales16 = prepare_scales(i); ++ mins = scb.shuffle(scales16); ++ scales[0] = MM256_SET1_M128I(scales16); ++ return -d*minv; ++ } ++ ++ inline static __m256i make_quants(const uint8_t * qs) { ++ return _mm256_set_epi32(iq3xxs_grid[qs[7]], iq3xxs_grid[qs[6]], iq3xxs_grid[qs[5]], iq3xxs_grid[qs[4]], ++ iq3xxs_grid[qs[3]], iq3xxs_grid[qs[2]], iq3xxs_grid[qs[1]], iq3xxs_grid[qs[0]]); ++ } ++ inline static void make4_unsigned(const uint8_t * qs, __m256i * values) { ++ values[0] = make_quants(qs+ 0); ++ values[1] = make_quants(qs+ 8); ++ values[2] = make_quants(qs+16); ++ values[3] = make_quants(qs+24); ++ } ++ ++ IQK_ALWAYS_INLINE void sign_2_values(const uint16_t * signs, __m256i * values) const { ++#if defined HAVE_FANCY_SIMD && defined __AVX512VPOPCNTDQ__ ++ esh.sign_2_values(MM256_SET_M128I(_mm_set1_epi32(signs[2] | (signs[3] << 16)), _mm_set1_epi32(signs[0] | (signs[1] << 16))), values); ++#else ++ esh.sign_value(signs[0] | (signs[1] << 16), values[0]); ++ esh.sign_value(signs[2] | (signs[3] << 16), values[1]); ++#endif ++ } ++ ++ inline void prepare(int i, int j) { ++ auto qs = x[i].qs + 32*j; ++ const uint16_t * signs = (const uint16_t *)(x[i].qs + QK_K/4) + 8*j; ++ make4_unsigned(qs, bits.values); ++ sign_2_values(signs+0, bits.values+0); ++ sign_2_values(signs+4, bits.values+2); ++ for (int k = 0; k < 4; ++k) bits.values[k] = _mm256_add_epi8(bits.values[k], min_value); ++ } ++ inline void prepare(int i, int j, const Q8<1>& q8, __m256i * q8_quants) { ++ for (int k = 0; k < 4; ++k) q8_quants[k] = q8.load_quants(0, i, 4*j+k); ++ auto qs = x[i].qs + 32*j; ++ const uint16_t * signs = (const uint16_t *)(x[i].qs + QK_K/4) + 8*j; ++ make4_unsigned(qs, bits.values); ++ sign_2_values(signs+0, q8_quants+0); ++ sign_2_values(signs+4, q8_quants+2); ++ } ++ inline void prepare(int i, int j, __m256i * q8_quants) { ++ auto qs = x[i].qs + 32*j; ++ const uint16_t * signs = (const uint16_t *)(x[i].qs + QK_K/4) + 8*j; ++ make4_unsigned(qs, bits.values); ++ sign_2_values(signs+0, q8_quants+0); ++ sign_2_values(signs+4, q8_quants+2); ++ } ++ ++ constexpr static int minv = 64; ++ ++ SimpleBits bits; ++ Scales8KBase scb; ++ EvenSignHelper esh; ++ const __m256i min_value = _mm256_set1_epi8(minv); ++ ++}; ++ ++#ifdef z_HAVE_FANCY_SIMD ++// Strangely enough, the following implementation makes PP ~6% slower and TG ~6% faster ++// compared to the vanilla AVX2 version below. ++struct IndexHelperIQ3S { ++ union index_t { ++ __m256i vec; ++ uint16_t val[16]; ++ }; ++ inline void make2(const uint8_t * qs, const uint8_t * qh, __m256i * values) const { ++ auto idx_l = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)qs)); ++ const __mmask16 * m16 = (const __mmask16 *)qh; ++ index_t idx; ++ idx.vec = _mm256_mask_add_epi16(idx_l, m16[0], idx_l, offset); ++ values[0] = _mm256_set_epi32(iq3s_grid[idx.val[ 7]], iq3s_grid[idx.val[ 6]], iq3s_grid[idx.val[ 5]], iq3s_grid[idx.val[ 4]], ++ iq3s_grid[idx.val[ 3]], iq3s_grid[idx.val[ 2]], iq3s_grid[idx.val[ 1]], iq3s_grid[idx.val[ 0]]); ++ values[1] = _mm256_set_epi32(iq3s_grid[idx.val[15]], iq3s_grid[idx.val[14]], iq3s_grid[idx.val[13]], iq3s_grid[idx.val[12]], ++ iq3s_grid[idx.val[11]], iq3s_grid[idx.val[10]], iq3s_grid[idx.val[ 9]], iq3s_grid[idx.val[ 8]]); ++ } ++ const __m256i offset = _mm256_set1_epi16(256); ++}; ++#else ++struct IndexHelperIQ3S { ++ union index_t { ++ __m256i vec; ++ uint32_t val[8]; ++ }; ++ inline void make2(const uint8_t * qs, const uint8_t * qh, __m256i * values) const { ++ index_t idx; ++ auto idx_l = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)qs)); ++ auto idx_h = _mm256_and_si256(_mm256_sllv_epi32(_mm256_set1_epi32(qh[0]), idx_shift), idx_mask); ++ idx.vec = _mm256_or_si256(idx_h, idx_l); ++ values[0] = _mm256_set_epi32(iq3s_grid[idx.val[7]], iq3s_grid[idx.val[6]], iq3s_grid[idx.val[5]], iq3s_grid[idx.val[4]], ++ iq3s_grid[idx.val[3]], iq3s_grid[idx.val[2]], iq3s_grid[idx.val[1]], iq3s_grid[idx.val[0]]); ++ idx_l = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(qs+8))); ++ idx_h = _mm256_and_si256(_mm256_sllv_epi32(_mm256_set1_epi32(qh[1]), idx_shift), idx_mask); ++ idx.vec = _mm256_or_si256(idx_h, idx_l); ++ values[1] = _mm256_set_epi32(iq3s_grid[idx.val[7]], iq3s_grid[idx.val[6]], iq3s_grid[idx.val[5]], iq3s_grid[idx.val[4]], ++ iq3s_grid[idx.val[3]], iq3s_grid[idx.val[2]], iq3s_grid[idx.val[1]], iq3s_grid[idx.val[0]]); ++ } ++ const __m256i idx_mask = _mm256_set1_epi32(256); ++ const __m256i idx_shift = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); ++}; ++#endif ++ ++struct DequantizerIQ3S final : public BaseDequantizer { ++ DequantizerIQ3S(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ constexpr static int num_blocks = 8; ++ ++ inline __m128i make_scales(int i, float& dd) const { ++ dd = GGML_FP16_TO_FP32(x[i].d); ++ uint32_t aux32[2]; ++ std::memcpy(aux32, x[i].scales, 4); ++ aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; ++ aux32[0] &= 0x0f0f0f0f; ++ auto scales8 = _mm_shuffle_epi8(_mm_loadl_epi64((const __m128i *)aux32), _mm_set1_epi64x(0x0703060205010400)); ++ auto scales16 = _mm256_castsi256_si128(_mm256_cvtepi8_epi16(scales8)); ++ return _mm_or_si128(_mm_slli_epi16(scales16, 1), _mm_set1_epi16(1)); ++ } ++ inline void new_block(int i, __m256i * scales) { ++ auto scales16 = make_scales(i, d); ++ scales[0] = MM256_SET1_M128I(scales16); ++ } ++ inline void new_block_f(int i, __m256 * scales) { ++ auto sc16 = make_scales(i, d); ++ auto scf = _mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(sc16))); ++ auto scf_l = _mm256_castps256_ps128(scf); ++ auto scf_h = _mm256_extractf128_ps(scf, 1); ++ scales[0] = _mm256_set_m128(scf_l, scf_l); ++ scales[1] = _mm256_set_m128(scf_h, scf_h); ++ scales[2] = _mm256_mul_ps(scf, _mm256_set1_ps(-minv)); ++ } ++ inline float new_block(int i, __m256i * scales, __m256i& mins) { ++ auto scales16 = make_scales(i, d); ++ mins = scb.shuffle(scales16); ++ scales[0] = MM256_SET1_M128I(scales16); ++ return -minv*d; ++ } ++ ++ inline void prepare(int i, int j) { ++ prepare_unsigned(i, j); ++ sh.sign_4_values((const uint16_t *)x[i].signs + 8*j, bits.values); ++ for (int k = 0; k < 4; ++k) bits.values[k] = _mm256_add_epi8(bits.values[k], min_value); ++ } ++ inline void prepare(int i, int j, const Q8<1>& q8, __m256i * q8_quants) { ++ prepare_unsigned(i, j); ++ for (int k = 0; k < 4; ++k) q8_quants[k] = q8.load_quants(0, i, 4*j+k); ++ sh.sign_4_values((const uint16_t *)x[i].signs + 8*j, q8_quants); ++ } ++ inline void prepare(int i, int j, __m256i * q8_quants) { ++ prepare_unsigned(i, j); ++ sh.sign_4_values((const uint16_t *)x[i].signs + 8*j, q8_quants); ++ } ++ ++ inline void prepare_unsigned(int i, int j) { ++ auto qs = x[i].qs + 32*j; ++ auto qh = x[i].qh + 4*j; ++ helper.make2(qs+ 0, qh+0, bits.values+0); ++ helper.make2(qs+16, qh+2, bits.values+2); ++ } ++ ++ constexpr static int minv = 16; ++ ++ SimpleBits bits; ++ SignHelper sh; ++ Scales8KBase scb; ++ IndexHelperIQ3S helper; ++ const __m256i min_value = _mm256_set1_epi8(minv); ++ ++}; ++ ++template ++inline void multiply_add_1(int j, const Bits& bits, const __m256i * scales, const __m256i * q8, __m256i * sumi) { ++ if (j == 0) { ++#ifdef HAVE_FANCY_SIMD ++ auto p1 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[0], q8[0]); ++ auto p2 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[1], q8[1]); ++ auto p3 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[2], q8[2]); ++ auto p4 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[3], q8[3]); ++ sumi[0] = _mm256_dpwssd_epi32(_mm256_setzero_si256(), scales[0], _mm256_packs_epi32(p1, p2)); ++ sumi[1] = _mm256_dpwssd_epi32(_mm256_setzero_si256(), scales[1], _mm256_packs_epi32(p3, p4)); ++#else ++ const __m256i p1 = _mm256_madd_epi16(scales[0], _mm256_maddubs_epi16(bits.values[0], q8[0])); ++ const __m256i p2 = _mm256_madd_epi16(scales[1], _mm256_maddubs_epi16(bits.values[1], q8[1])); ++ const __m256i p3 = _mm256_madd_epi16(scales[2], _mm256_maddubs_epi16(bits.values[2], q8[2])); ++ const __m256i p4 = _mm256_madd_epi16(scales[3], _mm256_maddubs_epi16(bits.values[3], q8[3])); ++ sumi[0] = _mm256_add_epi32(p1, p3); ++ sumi[1] = _mm256_add_epi32(p2, p4); ++#endif ++ } else { ++#ifdef HAVE_FANCY_SIMD ++ auto p1 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[0], q8[0]); ++ auto p2 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[1], q8[1]); ++ auto p3 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[2], q8[2]); ++ auto p4 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), bits.values[3], q8[3]); ++ sumi[0] = _mm256_dpwssd_epi32(sumi[0], scales[0], _mm256_packs_epi32(p1, p2)); ++ sumi[1] = _mm256_dpwssd_epi32(sumi[1], scales[1], _mm256_packs_epi32(p3, p4)); ++#else ++ const __m256i p1 = _mm256_madd_epi16(scales[0], _mm256_maddubs_epi16(bits.values[0], q8[0])); ++ const __m256i p2 = _mm256_madd_epi16(scales[1], _mm256_maddubs_epi16(bits.values[1], q8[1])); ++ const __m256i p3 = _mm256_madd_epi16(scales[2], _mm256_maddubs_epi16(bits.values[2], q8[2])); ++ const __m256i p4 = _mm256_madd_epi16(scales[3], _mm256_maddubs_epi16(bits.values[3], q8[3])); ++ sumi[0] = _mm256_add_epi32(sumi[0], _mm256_add_epi32(p1, p3)); ++ sumi[1] = _mm256_add_epi32(sumi[1], _mm256_add_epi32(p2, p4)); ++#endif ++ } ++} ++ ++template ++static void mul_mat_qX_K_q8_K_IQ_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ const int nb = n / QK_K; ++ Q8<1> q8(info); ++ Dequantizer deq(vx, bx); ++ __m256i scales[2]; ++ __m256i q8_quants[4]; ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ __m256 accd = _mm256_setzero_ps(); ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ __m256i sumi[2], all_scales[Dequantizer::num_blocks/8]; ++ deq.new_block(i, all_scales); ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ deq.prepare(i, j, q8, q8_quants); ++ if constexpr (Dequantizer::num_blocks == 8) { ++ set_scales_8_iq(j, all_scales[0], scales); ++ } else { ++ set_scales_16_iq(all_scales[j], scales); ++ } ++ multiply_add_1(j, deq.bits, scales, q8_quants, sumi); ++ } ++ accd = _mm256_fmadd_ps(_mm256_set1_ps(deq.d*q8.scale(0, i)), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi[0], sumi[1])), accd); ++ } ++ ++ info.store(ix, 0, hsum_float_8(accd)); ++ } ++} ++ ++// So, if I uncomment this function and the call to it in mul_mat_qX_K_q8_K_IQ_N() below, ++// PP performance improves by ~2-3% (when we have __AVX512VNNI__ and __AVX512VL__). ++// But TG performance for iq3_xs drops by 35%. Seriously? I mean, c'mon, ++// what does the compilation of mul_mat_qX_K_q8_K_IQ_1 (which gets invoked during TG) ++// have to do with the compilation of mul_mat_qX_K_q8_K_IQ_N (invoked during PP)? ++//template ++//inline void multiply_add_iq(const Bits& bits, const __m256i * scales, int j, int i, const Q8& q8, __m256i * sumi) { ++//#if defined(__AVX512VNNI__) && defined(__AVX512VL__) ++// for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++// sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 4*j+0))); ++// sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 4*j+1))); ++// sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 4*j+2))); ++// sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 4*j+3))); ++// } ++//#else ++// for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++// const __m256i p1 = _mm256_madd_epi16(scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 4*j+0))); ++// const __m256i p2 = _mm256_madd_epi16(scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 4*j+1))); ++// const __m256i p3 = _mm256_madd_epi16(scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 4*j+2))); ++// const __m256i p4 = _mm256_madd_epi16(scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 4*j+3))); ++// sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p1, p3)); ++// sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p2, p4)); ++// } ++//#endif ++//} ++ ++template ++static void mul_mat_qX_K_q8_K_IQ_N(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ const int nb = n / QK_K; ++ Q8 q8(info); ++ Dequantizer deq(vx, bx); ++ __m256i scales[4]; ++ __m256 accd[nrc_y]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ __m256i sumi[nrc_y], all_scales[Dequantizer::num_blocks/8]; ++ //for (int iy = 0; iy < nrc_y; ++iy) sumi[iy] = _mm256_setzero_si256(); ++ __m256i mins; ++ float dmin = deq.new_block(i, all_scales, mins); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto bsums = q8.load_bsums(iy, i); ++ auto prod = _mm256_madd_epi16(mins, bsums); ++ accd[iy] = _mm256_fmadd_ps(_mm256_set1_ps(dmin*q8.scale(iy, i)), _mm256_cvtepi32_ps(prod), accd[iy]); ++ } ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ deq.prepare(i, j); ++ if constexpr (Dequantizer::num_blocks == 8) { ++ set_scales_8(all_scales[0], j, scales); ++ } else { ++ set_scales_16(all_scales[j], scales); ++ } ++ //multiply_add_iq(deq.bits, scales, j, i, q8, sumi); ++ multiply_add(deq.bits, scales, j, i, q8, sumi); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ const __m256 vd = _mm256_set1_ps(deq.d*q8.scale(iy, i)); ++ accd[iy] = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi[iy]), accd[iy]); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, hsum_float_8(accd[iy])); ++ } ++ } ++} ++ ++template ++static void mul_mat_qX_K_q8_K_IQ(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++#ifdef HAVE_FANCY_SIMD ++ if constexpr (nrc_y == 1) { ++ mul_mat_qX_K_q8_K_IQ_1(n, vx, bx, info, nrc_x); ++ } else { ++ mul_mat_qX_K_q8_K_IQ_N(n, vx, bx, info, nrc_x); ++ } ++#else ++ mul_mat_qX_K_q8_K_IQ_N(n, vx, bx, info, nrc_x); ++#endif ++} ++ ++template void set_functions(mul_mat_t * funcs) { ++ funcs[0] = mul_mat_qX_K_q8_K_IQ; ++ funcs[1] = mul_mat_qX_K_q8_K_IQ; ++ funcs[2] = mul_mat_qX_K_q8_K_IQ; ++ funcs[3] = mul_mat_qX_K_q8_K_IQ; ++ funcs[4] = mul_mat_qX_K_q8_K_IQ; ++ funcs[5] = mul_mat_qX_K_q8_K_IQ; ++ funcs[6] = mul_mat_qX_K_q8_K_IQ; ++ funcs[7] = mul_mat_qX_K_q8_K_IQ; ++} ++ ++} // namespace ++ ++bool iqk_set_kernels_iquants(int typeA, mul_mat_t * kernels) { ++ switch (typeA) { ++ case GGML_TYPE_IQ2_XXS: set_functions(kernels); break; ++ case GGML_TYPE_IQ2_XS: set_functions(kernels); break; ++ case GGML_TYPE_IQ2_S: set_functions(kernels); break; ++ case GGML_TYPE_IQ3_XXS: set_functions(kernels); break; ++ case GGML_TYPE_IQ3_S: set_functions(kernels); break; ++ default: return false; ++ } ++ return true; ++} ++ ++#else ++// --------------------------------- __aarch64__ -------------------------------------- ++ ++namespace { ++ ++inline int32x4x4_t make_wider(const int16x8x2_t& scales16) { ++ int32x4x4_t scales = { ++ vmovl_s16(vget_low_s16 (scales16.val[0])), ++ vmovl_s16(vget_high_s16(scales16.val[0])), ++ vmovl_s16(vget_low_s16 (scales16.val[1])), ++ vmovl_s16(vget_high_s16(scales16.val[1])), ++ }; ++ return scales; ++} ++ ++struct SimpleBits { ++ uint8x16x4_t b1; ++ uint8x16x4_t b2; ++}; ++ ++inline int32x4x2_t prepare_scales_8(const uint32x4_t& v1, const uint32x4_t& v2) { ++ int32x4x2_t scales; ++ scales.val[0] = vreinterpretq_s32_u32(vorrq_u32(vshlq_n_u32(vshrq_n_u32(v1, 28), 1), vdupq_n_u32(1))); ++ scales.val[1] = vreinterpretq_s32_u32(vorrq_u32(vshlq_n_u32(vshrq_n_u32(v2, 28), 1), vdupq_n_u32(1))); ++ return scales; ++} ++ ++inline void apply_signs_2(uint8x16_t * b, const uint64_t * signs, uint32_t sidx) { ++ auto s1 = vcombine_s8(vld1_s8((const int8_t *)(signs + ((sidx >> 0) & 127))), vld1_s8((const int8_t *)(signs + ((sidx >> 7) & 127)))); ++ auto s2 = vcombine_s8(vld1_s8((const int8_t *)(signs + ((sidx >>14) & 127))), vld1_s8((const int8_t *)(signs + ((sidx >>21) & 127)))); ++ b[0] = vreinterpretq_u8_s8(vmulq_s8(vreinterpretq_s8_u8(b[0]), s1)); ++ b[1] = vreinterpretq_u8_s8(vmulq_s8(vreinterpretq_s8_u8(b[1]), s2)); ++} ++ ++struct DequantizerIQ2XXS final : public BaseDequantizer { ++ DequantizerIQ2XXS(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& /*q8*/, float32x4_t * /*acc*/) { ++ d = 0.125f * GGML_FP16_TO_FP32(x[i].d); ++ ++ auto tmp = vld1q_u32_x4((const uint32_t *)x[i].qs); ++ data.val[0] = vuzp1q_u32(tmp.val[0], tmp.val[1]); // codebook indices for blocks 0...3 ++ data.val[1] = vuzp2q_u32(tmp.val[0], tmp.val[1]); // scales and signs for blocks 0...3 ++ data.val[2] = vuzp1q_u32(tmp.val[2], tmp.val[3]); // codebook indices for blocks 4...7 ++ data.val[3] = vuzp2q_u32(tmp.val[2], tmp.val[3]); // scales and signs for blocks 4...7 ++ ++ return prepare_scales_8(data.val[1], data.val[3]); ++ } ++ ++ static inline void prepare2(uint8x16_t * b, const uint8_t * idx, const uint64_t * signs, uint32_t sidx) { ++ b[0] = vreinterpretq_u8_u64(uint64x2_t{iq2xxs_grid[idx[0]], iq2xxs_grid[idx[1]]}); ++ b[1] = vreinterpretq_u8_u64(uint64x2_t{iq2xxs_grid[idx[2]], iq2xxs_grid[idx[3]]}); ++ apply_signs_2(b, signs, sidx); ++ } ++ ++ inline void prepare(int /*i*/, int j) { ++ const uint8_t * idx = (const uint8_t *)(data.val + 2*j); ++ const uint32_t * sidx = (const uint32_t *)(data.val + 2*j+1); ++ prepare2(bits.b1.val + 0, idx, keven_signs, sidx[0]); idx += 4; ++ prepare2(bits.b1.val + 2, idx, keven_signs, sidx[1]); idx += 4; ++ prepare2(bits.b2.val + 0, idx, keven_signs, sidx[2]); idx += 4; ++ prepare2(bits.b2.val + 2, idx, keven_signs, sidx[3]); ++ } ++ ++ uint32x4x4_t data; ++ SimpleBits bits; ++ ++}; ++ ++inline int32x4x4_t prepare_4bit_scales16(const uint8_t * sc) { ++ auto aux = vld1_u8(sc); ++ auto scales_l = vand_u8(aux, vdup_n_u8(0xf)); ++ auto scales_h = vshr_n_u8(aux, 4); ++ auto aux1 = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h)); ++ ++ auto scales8 = vreinterpretq_s8_u8(vorrq_u8(vshlq_n_u8(aux1, 1), vdupq_n_u8(1))); ++ int16x8x2_t scales16 = { vmovl_s8(vget_low_s8(scales8)), vmovl_s8(vget_high_s8(scales8)) }; ++ return make_wider(scales16); ++} ++ ++struct DequantizerIQ2XS final : public BaseDequantizer { ++ DequantizerIQ2XS(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& /*q8*/, float32x4_t * /*acc*/) { ++ d = 0.125f * GGML_FP16_TO_FP32(x[i].d); ++ return prepare_4bit_scales16(x[i].scales); ++ } ++ ++ inline static uint8x16_t make1(const uint16_t * qs) { ++ auto b = vcombine_u8(vld1_u8((const uint8_t *)(iq2xs_grid + (qs[0] & 511))), vld1_u8((const uint8_t *)(iq2xs_grid + (qs[1] & 511)))); ++ auto s = vcombine_s8(vld1_s8((const int8_t *)(keven_signs + (qs[0] >> 9))), vld1_s8((const int8_t *)(keven_signs + (qs[1] >> 9)))); ++ return vreinterpretq_u8_s8(vmulq_s8(vreinterpretq_s8_u8(b), s)); ++ } ++ ++ inline static void make4(const uint16_t * qs, uint8x16_t * b) { ++ b[0] = make1(qs + 0); ++ b[1] = make1(qs + 2); ++ b[2] = make1(qs + 4); ++ b[3] = make1(qs + 6); ++ } ++ ++ inline void prepare(int i, int j) { ++ make4(x[i].qs + 16*j + 0, bits.b1.val); ++ make4(x[i].qs + 16*j + 8, bits.b2.val); ++ } ++ ++ SimpleBits bits; ++ ++ ++}; ++ ++struct DequantizerIQ2S final : public BaseDequantizer { ++ DequantizerIQ2S(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& /*q8*/, float32x4_t * /*acc*/) { ++ d = 0.125f * GGML_FP16_TO_FP32(x[i].d); ++ return prepare_4bit_scales16(x[i].scales); ++ } ++ ++ static inline void make4(SignHelper& sh, const uint8x16_t& signs16, const uint8_t * qs, const uint8_t * qh, uint8x16_t * b) { ++ uint32_t aux32[2]; ++ const uint16_t * aux16 = (const uint16_t *)aux32; ++ for (int k = 0; k < 2; ++k) { ++ aux32[1] = (qh[k] << 4) | (qh[k] << 18); ++ aux32[0] = (aux32[1] << 4) & 0x03000300; ++ aux32[1] &= 0x03000300; ++ b[2*k+0] = vcombine_u8(vld1_u8((const uint8_t *)(iq2s_grid + (qs[4*k+0] | aux16[0]))), ++ vld1_u8((const uint8_t *)(iq2s_grid + (qs[4*k+1] | aux16[1])))); ++ sh.apply_signs_1(b+2*k+0, signs16); ++ ++ b[2*k+1] = vcombine_u8(vld1_u8((const uint8_t *)(iq2s_grid + (qs[4*k+2] | aux16[2]))), ++ vld1_u8((const uint8_t *)(iq2s_grid + (qs[4*k+3] | aux16[3])))); ++ sh.apply_signs_1(b+2*k+1, signs16); ++ } ++ } ++ ++ inline void prepare(int i, int j) { ++ ++ const auto * qs = x[i].qs + 16*j; ++ const auto * qh = x[i].qh + 4*j; ++ const auto signs16 = vld1q_u8(qs + QK_K/8); ++ ++ sh.init(); ++ make4(sh, signs16, qs+0, qh+0, bits.b1.val); ++ make4(sh, signs16, qs+8, qh+2, bits.b2.val); ++ } ++ ++ SimpleBits bits; ++ SignHelper sh; ++ ++ ++}; ++ ++struct DequantizerIQ3XXS final : public BaseDequantizer { ++ DequantizerIQ3XXS(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& /*q8*/, float32x4_t * /*acc*/) { ++ d = 0.25f * GGML_FP16_TO_FP32(x[i].d); ++ gas = vld1q_u32_x2((const uint32_t *)(x[i].qs + QK_K/4)); ++ return prepare_scales_8(gas.val[0], gas.val[1]); ++ } ++ ++ inline static void make2(const uint8_t * q3, uint32_t sidx, uint8x16_t * b) { ++ b[0] = vreinterpretq_u8_u32(uint32x4_t{iq3xxs_grid[q3[0]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[3]]}); ++ b[1] = vreinterpretq_u8_u32(uint32x4_t{iq3xxs_grid[q3[4]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[7]]}); ++ apply_signs_2(b, keven_signs, sidx); ++ } ++ inline void prepare(int i, int j) { ++ const auto * q3 = x[i].qs + 32*j; ++ const auto * signs = (const uint32_t *)(gas.val + j); ++ make2(q3, signs[0], bits.b1.val + 0); q3 += 8; ++ make2(q3, signs[1], bits.b1.val + 2); q3 += 8; ++ make2(q3, signs[2], bits.b2.val + 0); q3 += 8; ++ make2(q3, signs[3], bits.b2.val + 2); ++ } ++ ++ SimpleBits bits; ++ uint32x4x2_t gas; ++ ++}; ++ ++struct DequantizerIQ3S final : public BaseDequantizer { ++ DequantizerIQ3S(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& /*q8*/, float32x4_t * /*acc*/) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ uint32_t scales32[2]; ++ std::memcpy(scales32, x[i].scales, 4); ++ scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101; ++ scales32[0] = ((scales32[0] & 0x0f0f0f0f) << 1) | 0x01010101; ++ auto scales8 = vld1_u8((const uint8_t *)scales32); // 0, 2, 4, 6, 1, 3, 5, 7 ++ scales8 = vtbl1_u8(scales8, vreinterpret_u8_u64(vdup_n_u64(0x0703060205010400))); ++ auto scales16 = vreinterpretq_s16_u16(vmovl_u8(scales8)); ++ int32x4x2_t scales; ++ scales.val[0] = vmovl_s16(vget_low_s16(scales16)); ++ scales.val[1] = vmovl_s16(vget_high_s16(scales16)); ++ return scales; ++ } ++ ++ static inline void make2(SignHelper& sh, const uint8x16_t& signs16, const uint16x8_t& idx_l, uint8_t qh, ++ const int8x16_t& hshift, uint8x16_t * b) { ++ auto vindex = vorrq_u16(idx_l, vandq_u16(vshlq_u16(vdupq_n_u16(qh), hshift), vdupq_n_u16(256))); ++ const uint16_t * idx = (const uint16_t *)&vindex; ++ b[0] = vreinterpretq_u8_u32(uint32x4_t{iq3s_grid[idx[0]], iq3s_grid[idx[1]], iq3s_grid[idx[2]], iq3s_grid[idx[3]]}); ++ b[1] = vreinterpretq_u8_u32(uint32x4_t{iq3s_grid[idx[4]], iq3s_grid[idx[5]], iq3s_grid[idx[6]], iq3s_grid[idx[7]]}); ++ sh.apply_signs_1(b+0, signs16); ++ sh.apply_signs_1(b+1, signs16); ++ } ++ static inline void make4(SignHelper& sh, const uint8x16_t& signs16, const uint8_t * qs, const uint8_t * qh, ++ const int8x16_t& hshift, uint8x16_t * b) { ++ auto idx_l = vld1q_u8(qs); ++ make2(sh, signs16, vmovl_u8(vget_low_u8 (idx_l)), qh[0], hshift, b+0); ++ make2(sh, signs16, vmovl_u8(vget_high_u8(idx_l)), qh[1], hshift, b+2); ++ } ++ ++ inline void prepare(int i, int j) { ++ ++ static const int16_t k_shift[8] = {8, 7, 6, 5, 4, 3, 2, 1}; ++ const auto hshift = vld1q_s16(k_shift); ++ ++ const auto * qs = x[i].qs + 32*j; ++ const auto * qh = x[i].qh + 4*j; ++ const auto signs16 = vld1q_u8(x[i].signs + 16*j); ++ ++ sh.init(); ++ make4(sh, signs16, qs+ 0, qh+0, hshift, bits.b1.val); ++ make4(sh, signs16, qs+16, qh+2, hshift, bits.b2.val); ++ } ++ ++ SimpleBits bits; ++ SignHelper sh; ++ uint32x4x2_t gas; ++ ++}; ++ ++} // namespace ++ ++bool iqk_set_kernels_iquants(int typeA, mul_mat_t * kernels) { ++ switch (typeA) { ++ case GGML_TYPE_IQ2_XXS: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerIQ2XXS, kernels); break; ++ case GGML_TYPE_IQ2_XS: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerIQ2XS, kernels); break; ++ case GGML_TYPE_IQ2_S: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerIQ2S, kernels); break; ++ case GGML_TYPE_IQ3_XXS: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerIQ3XXS, kernels); break; ++ case GGML_TYPE_IQ3_S: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerIQ3S, kernels); break; ++ default: return false; ++ } ++ return true; ++} ++ ++#endif ++ ++#endif // IQK_IMPLEMENT +diff --git a/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp b/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp +new file mode 100644 +index 00000000..2dd6980a +--- /dev/null ++++ b/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp +@@ -0,0 +1,1162 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++// Adapted from ik_llama.cpp (https://github.com/ikawrakow/ik_llama.cpp) commit 009dce19, ++// ggml/src/iqk/{iqk_config.h, iqk_common.h, iqk_gemm_kquants.cpp, iqk_gemm_iquants.cpp, iqk_gemm_1bit.cpp, iqk_mul_mat.cpp}. ++// Only the kernels that read the upstream ggml block layouts with upstream Q8_K activations are kept. ++// ++// Permission is hereby granted, free of charge, to any person obtaining a copy ++// of this software and associated documentation files (the "Software"), to deal ++// in the Software without restriction, including without limitation the rights ++// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell ++// copies of the Software, and to permit persons to whom the Software is ++// furnished to do so, subject to the following conditions: ++// ++// The above copyright notice and this permission notice shall be included in all ++// copies or substantial portions of the Software. ++// ++// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ++// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++// SOFTWARE. ++// ++// k-quants (Q2_K, Q3_K, Q4_K, Q5_K, Q6_K) and IQ4_XS times Q8_K (from iqk_gemm_kquants.cpp). ++ ++#include "iqk_common.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++#ifdef __x86_64__ ++ ++namespace { ++ ++// Handles q4_K and q5_K scales/mins ++struct Scales8K { ++ template ++ inline __m256i process_mins_and_scales(const uint8_t * data, float c, int i, const Q8& q8, __m256 * accd) { ++ make_q4_scales(data, utmp); ++ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0])); ++ const __m128i mins128 = _mm256_extracti128_si256(mins_and_scales, 1); ++ accum_mins(mins128, q8, i, c, accd); ++ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0); ++ return MM256_SET1_M128I(sc128); ++ } ++#ifdef HAVE_FANCY_SIMD ++ template ++ inline __m512i process_mins_and_scales_64(const uint8_t * data, float c, int i, const Q8& q8, __m256 * accd) { ++ auto scales = process_mins_and_scales(data, c, i, q8, accd); ++ return _mm512_inserti32x8(_mm512_castsi256_si512(scales), scales, 1); ++ } ++#endif ++ template ++ inline void accum_mins(const __m128i& mins128, const Q8& q8, int i, float c, __m256 * accd) const { ++ base.accum_mins(mins128, q8, i, c, accd); ++ } ++#ifdef HAVE_FANCY_SIMD ++ const __m512i shuffles512[2] = { ++ _mm512_set_epi64(0x0706070607060706, 0x0302030203020302, 0x0706070607060706, 0x0302030203020302, ++ 0x0504050405040504, 0x0100010001000100, 0x0504050405040504, 0x0100010001000100), ++ _mm512_set_epi64(0x0f0e0f0e0f0e0f0e, 0x0b0a0b0a0b0a0b0a, 0x0f0e0f0e0f0e0f0e, 0x0b0a0b0a0b0a0b0a, ++ 0x0d0c0d0c0d0c0d0c, 0x0908090809080908, 0x0d0c0d0c0d0c0d0c, 0x0908090809080908) ++ }; ++#endif ++ Scales8KBase base; ++ ++ uint32_t utmp[4]; ++}; ++ ++template ++inline void process_mins_16(const __m256i& all_scales, const Q8& q8, int i, float d, __m256 * accm) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i prod = _mm256_madd_epi16(all_scales, q8.load_bsums(iy, i)); ++ accm[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d * q8.scale(iy, i)), _mm256_cvtepi32_ps(prod), accm[iy]); ++ } ++} ++inline void prepare_scales_16(const __m256i& all_scales, __m256i * scales) { ++ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0); ++ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1); ++ scales[0] = MM256_SET1_M128I(l_scales); ++ scales[1] = MM256_SET1_M128I(h_scales); ++} ++ ++// Handles q3_K scales ++struct ScaleQ3 { ++ inline __m128i make_scales(const uint16_t * s8) const { ++ const uint16_t * scales16 = (const uint16_t *)s8; ++ uint32_t aux0 = scales16[0] | (scales16[1] << 16); ++ uint32_t aux1 = scales16[2] | (scales16[3] << 16); ++ uint32_t aux2 = scales16[4] | (scales16[5] << 16); ++ __m128i scales128 = _mm_set_epi32( ++ ((aux1 >> 4) & 0x0f0f0f0f) | ((aux2 >> 2) & 0x30303030), ++ ((aux0 >> 4) & 0x0f0f0f0f) | ((aux2 >> 0) & 0x30303030), ++ (aux1 & 0x0f0f0f0f) | ((aux2 << 2) & 0x30303030), ++ (aux0 & 0x0f0f0f0f) | ((aux2 << 4) & 0x30303030)); ++ return _mm_add_epi8(scales128, m32); ++ } ++ const __m128i m32 = _mm_set1_epi8(-32); ++}; ++ ++struct Scale16 { ++ inline void make_scales(const __m128i& scales8, __m512i * scales) const { ++ auto all_scales8 = MM256_SET1_M128I(scales8); ++ auto scales1 = _mm256_shuffle_epi8(all_scales8, shuffle1); ++ auto scales2 = _mm256_shuffle_epi8(all_scales8, shuffle2); ++ scales[0] = _mm512_cvtepi8_epi16(scales1); ++ scales[1] = _mm512_cvtepi8_epi16(scales2); ++ } ++ template ++ inline void process_mins_and_scales(int i, float c, const __m128i& mins8, const __m128i& scales8, ++ const Q8& q8, __m256 * accm, __m512i * scales) const { ++ process_mins_16(_mm256_cvtepi8_epi16(mins8), q8, i, c, accm); ++ make_scales(scales8, scales); ++ } ++ const __m256i shuffle1 = _mm256_set_epi32(0x07070707, 0x03030303, 0x06060606, 0x02020202, ++ 0x05050505, 0x01010101, 0x04040404, 0x00000000); ++ const __m256i shuffle2 = _mm256_set_epi32(0x0f0f0f0f, 0x0b0b0b0b, 0x0e0e0e0e, 0x0a0a0a0a, ++ 0x0d0d0d0d, 0x09090909, 0x0c0c0c0c, 0x08080808); ++}; ++ ++template ++inline void process_mins_and_scales_16(const __m128i& scales128, const Q8& q8, int i, float d, ++ __m256 * accm, __m256i * scales) { ++ const __m256i all_scales = _mm256_cvtepi8_epi16(scales128); ++ process_mins_16(all_scales, q8, i, d, accm); ++ prepare_scales_16(all_scales, scales); ++} ++ ++inline __m256i get_scale_shuffle_8(int i) { ++ return _mm256_set1_epi16((2*i) | ((2*i+1) << 8)); ++} ++ ++inline void set_scales_8(const __m256i& all_scales, int j, __m256i * scales) { ++ scales[0] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+0)); ++ scales[1] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+1)); ++ scales[2] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+2)); ++ scales[3] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+3)); ++} ++ ++inline __m256i get_scale_shuffle_16(int i) { ++ static const uint8_t k_shuffle[128] = { ++ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, ++ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, ++ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11, ++ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15, ++ }; ++ return _mm256_loadu_si256((const __m256i*)k_shuffle + i); ++} ++ ++inline void set_scales_16(const __m256i& all_scales, __m256i * scales) { ++ scales[0] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(0)); ++ scales[1] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(1)); ++ scales[2] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(2)); ++ scales[3] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(3)); ++} ++ ++struct ScaleIQ4XS { ++ inline __m128i make_scales(const uint32_t scales_l, const uint16_t scales_h) { ++ uint32_t tmp32 = scales_h | (scales_h << 14); ++ const __m128i sh = _mm_slli_epi16(_mm_and_si128(_mm_srlv_epi32(_mm_set1_epi32(tmp32), hshift), hmask), 4); ++ const __m128i sl = _mm_and_si128(_mm_srlv_epi32(_mm_set1_epi32(scales_l), lshift), lmask); ++ return _mm_add_epi16(_mm_or_si128(sh, _mm_cvtepi8_epi16(_mm_shuffle_epi8(sl, lshuffle))), m32); ++ } ++ const __m128i hshift = _mm_set_epi32(12, 8, 4, 0); ++ const __m128i lshift = _mm_set_epi32(4, 0, 4, 0); ++ const __m128i hmask = _mm_set1_epi16(0x03); ++ const __m128i lmask = _mm_set1_epi8(0xf); ++ const __m128i lshuffle = _mm_set_epi32(0x07030602, 0x05010400, 0x07030602, 0x05010400); ++ const __m128i m32 = _mm_set1_epi16(-32); ++}; ++ ++#ifdef HAVE_FANCY_SIMD ++//====================================== Zen4 ================================================== ++ ++struct HighBit5 { ++ inline void apply(const uint8_t * h, Q4Bits& bits) { ++ auto hbits256 = _mm256_loadu_si256((const __m256i *)h); ++ auto hbits = _mm512_inserti32x8(_mm512_castsi256_si512(hbits256), _mm256_srli_epi16(hbits256, 1), 1); ++ bits.values[0] = _mm512_or_si512(bits.values[0], _mm512_and_si512(_mm512_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm512_or_si512(bits.values[1], _mm512_and_si512(_mm512_slli_epi16(hbits, 2), mh)); ++ bits.values[2] = _mm512_or_si512(bits.values[2], _mm512_and_si512(hbits, mh)); ++ bits.values[3] = _mm512_or_si512(bits.values[3], _mm512_and_si512(_mm512_srli_epi16(hbits, 2), mh)); ++ } ++ const __m512i mh = _mm512_set1_epi8(0x10); ++}; ++ ++struct HighBit3 { ++ inline void apply(const uint8_t * h, Q2Bits& bits) { ++ auto hbits256 = _mm256_loadu_si256((const __m256i *)h); ++ auto hbits = _mm512_inserti32x8(_mm512_castsi256_si512(hbits256), _mm256_srli_epi16(hbits256, 1), 1); ++ bits.values[0] = _mm512_or_si512(bits.values[0], _mm512_and_si512(_mm512_slli_epi16(hbits, 2), mh)); ++ bits.values[1] = _mm512_or_si512(bits.values[1], _mm512_and_si512(hbits, mh)); ++ bits.values[2] = _mm512_or_si512(bits.values[2], _mm512_and_si512(_mm512_srli_epi16(hbits, 2), mh)); ++ bits.values[3] = _mm512_or_si512(bits.values[3], _mm512_and_si512(_mm512_srli_epi16(hbits, 4), mh)); ++ } ++ const __m512i mh = _mm512_set1_epi8(0x04); ++}; ++ ++ ++template ++inline void compute_block(int iy, int i, float d, const Q8& q8, const __m512i * values, const __m512i * scales, __m512 * accd) { ++ const __m512i p1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[0], q8.load_quants64(iy, i, 0)); ++ const __m512i p2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[1], q8.load_quants64(iy, i, 1)); ++ const __m512i p3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[2], q8.load_quants64(iy, i, 2)); ++ const __m512i p4 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[3], q8.load_quants64(iy, i, 3)); ++ auto sumi = _mm512_dpwssd_epi32(_mm512_setzero_si512(), scales[0], _mm512_packs_epi32(p1, p2)); ++ sumi = _mm512_dpwssd_epi32(sumi, scales[1], _mm512_packs_epi32(p3, p4)); ++ accd[iy] = _mm512_fmadd_ps(_mm512_set1_ps(d*q8.scale(iy, i)), _mm512_cvtepi32_ps(sumi), accd[iy]); ++} ++ ++struct DequantizerQ2K final : public BaseDequantizer { ++ DequantizerQ2K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ bits.prepare(x[i].qs); ++ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales); ++ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4); ++ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); ++ sc16.process_mins_and_scales(i, -GGML_FP16_TO_FP32(x[i].data.data.dmin), mins8, scales8, q8, accm, scales); ++ } ++ ++ Q2Bits bits; ++ Scale16 sc16; ++ const __m128i m4 = _mm_set1_epi8(0xf); ++ ++}; ++ ++struct DequantizerQ3K final : public BaseDequantizer { ++ DequantizerQ3K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ bits.prepare(x[i].qs); ++ hbits.apply(x[i].hmask, bits); ++ auto scales128 = sc3.make_scales((const uint16_t *)x[i].scales); ++ sc16.process_mins_and_scales(i, -4.f*d, scales128, scales128, q8, accm, scales); ++ } ++ ++ Q2Bits bits; ++ HighBit3 hbits; ++ ScaleQ3 sc3; ++ Scale16 sc16; ++ const __m128i m4 = _mm_set1_epi8(0xf); ++ const __m128i m32 = _mm_set1_epi8(-32); ++}; ++ ++struct DequantizerQ4K final : public BaseDequantizer { ++ DequantizerQ4K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accd, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ bits.prepare(x[i].qs); ++ auto all_scales = s8k.process_mins_and_scales_64(x[i].scales, -GGML_FP16_TO_FP32(x[i].data.data.dmin), i, q8, accd); ++ scales[0] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[0]); ++ scales[1] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[1]); ++ } ++ ++ Q4Bits bits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ5K final : public BaseDequantizer { ++ DequantizerQ5K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accd, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ bits.prepare(x[i].qs); ++ hbits.apply(x[i].qh, bits); ++ auto all_scales = s8k.process_mins_and_scales_64(x[i].scales, -GGML_FP16_TO_FP32(x[i].data.data.dmin), i, q8, accd); ++ scales[0] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[0]); ++ scales[1] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[1]); ++ } ++ ++ Q4Bits bits; ++ HighBit5 hbits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ6K final : public BaseDequantizer { ++ DequantizerQ6K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ bits.prepare64(x[i].ql); ++ add_high_bits(x[i].qh, bits); ++ auto scales128 = _mm_loadu_si128((const __m128i *)x[i].scales); ++ sc16.process_mins_and_scales(i, -32.f*d, scales128, scales128, q8, accm, scales); ++ } ++ ++ inline void add_high_bits(const uint8_t * qh, Q4Bits& bits) const { ++ auto hbits = _mm512_loadu_si512((const __m512i *)qh); ++ auto tmp1 = _mm512_and_si512(_mm512_slli_epi16(hbits, 4), mh); ++ auto tmp2 = _mm512_and_si512(_mm512_slli_epi16(hbits, 2), mh); ++ bits.values[0] = _mm512_or_si512(bits.values[0], _mm512_permutex2var_epi64(tmp1, bits.perm.permute1, tmp2)); ++ bits.values[2] = _mm512_or_si512(bits.values[2], _mm512_permutex2var_epi64(tmp1, bits.perm.permute2, tmp2)); ++ tmp1 = _mm512_and_si512(hbits, mh); ++ tmp2 = _mm512_and_si512(_mm512_srli_epi16(hbits, 2), mh); ++ bits.values[1] = _mm512_or_si512(bits.values[1], _mm512_permutex2var_epi64(tmp1, bits.perm.permute1, tmp2)); ++ bits.values[3] = _mm512_or_si512(bits.values[3], _mm512_permutex2var_epi64(tmp1, bits.perm.permute2, tmp2)); ++ } ++ ++ Q4Bits bits; ++ HighBit3 hbits; ++ Scale16 sc16; ++ ++ const __m512i mh = _mm512_set1_epi8(0x30); ++ ++}; ++ ++struct DequantizerIQ4XS final : public BaseDequantizer { ++ DequantizerIQ4XS(const void * vx, size_t bx) : BaseDequantizer(vx, bx), values(load_iq4nl_values_512()) {} ++ inline __m512 new_block(int i, __m512i& mins) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ prepare(x[i].qs); ++ auto scales128 = siq4.make_scales(*(const uint32_t *)x[i].scales_l, x[i].scales_h); ++ mins = dpbusd_permute_epi16(_mm256_set1_epi16(-128)); ++ return dpbusd_scales(d, s8k.shuffle(scales128)); ++ } ++ inline void prepare(const uint8_t * q4) { ++ bits.prepare64(q4); ++ // We now have in bits.valuse[0]: 0...15, 32...47, 64...79, 96...111 ++ // bits.valuse[1]: 16..31, 48...63, 80...95, 112..127 ++ // etc. ++ auto tmp = _mm512_permutex2var_epi64(bits.values[0], permute1, bits.values[1]); ++ bits.values[1] = _mm512_shuffle_epi8(values, _mm512_permutex2var_epi64(bits.values[0], permute2, bits.values[1])); ++ bits.values[0] = _mm512_shuffle_epi8(values, tmp); ++ tmp = _mm512_permutex2var_epi64(bits.values[2], permute1, bits.values[3]); ++ bits.values[3] = _mm512_shuffle_epi8(values, _mm512_permutex2var_epi64(bits.values[2], permute2, bits.values[3])); ++ bits.values[2] = _mm512_shuffle_epi8(values, tmp); ++ } ++ ++ Q4Bits bits; ++ Scales8KBase s8k; ++ ScaleIQ4XS siq4; ++ const __m512i values; ++ const __m512i permute1 = _mm512_set_epi64(11, 10, 3, 2, 9, 8, 1, 0); ++ const __m512i permute2 = _mm512_set_epi64(15, 14, 7, 6, 13, 12, 5, 4); ++}; ++ ++template ++static void mul_mat_qX_K_q8_K_AVX512_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ constexpr int k_nx = 2; ++ ++ Q8<1> q8(info); ++ ++ Dequantizer deq1(vx, bx); ++ Dequantizer deq2(vx, bx); ++ ++ Dequantizer * deq[k_nx]; ++ deq[0] = &deq1; ++ deq[1] = &deq2; ++ ++ __m512i scales[2*k_nx]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ auto accd = _mm512_setzero_ps(); ++ auto accm = _mm256_setzero_ps(); ++ ++ for (int kx = 0; kx < k_nx; ++kx) deq[kx]->new_row(ix); ++ ++ for (int i = 0; i < nb/k_nx; ++i) { ++ ++ for (int kx = 0; kx < k_nx; ++kx) deq[kx]->new_block(k_nx*i+kx, q8, &accm, scales+2*kx); ++ ++ for (int kx = 0; kx < k_nx; ++kx) { ++ compute_block(0, k_nx*i+kx, deq[kx]->d, q8, deq[kx]->bits.values, scales+2*kx, &accd); ++ } ++ ++ } ++ if (2*(nb/2) < nb) { ++ int i0 = 2*(nb/2); ++ deq[0]->new_block(i0, q8, &accm, scales); ++ compute_block(0, i0, deq[0]->d, q8, deq[0]->bits.values, scales, &accd); ++ } ++ ++ auto sum256 = _mm256_add_ps(_mm512_castps512_ps256(accd), _mm512_extractf32x8_ps(accd, 1)); ++ info.store(ix, 0, hsum_float_8(_mm256_add_ps(accm, sum256))); ++ } ++} ++ ++template ++static void mul_mat_qX_K_q8_K_AVX512(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ __m256 accm[nrc_y]; ++ __m512 accd[nrc_y]; ++ __m512i scales[2]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm512_setzero_ps(); ++ for (int iy = 0; iy < nrc_y; ++iy) accm[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ deq.new_block(i, q8, accm, scales); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ const __m512i p1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[0], q8.load_quants64(iy, i, 0)); ++ const __m512i p2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[1], q8.load_quants64(iy, i, 1)); ++ const __m512i p3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[2], q8.load_quants64(iy, i, 2)); ++ const __m512i p4 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[3], q8.load_quants64(iy, i, 3)); ++ auto sumi = _mm512_dpwssd_epi32(_mm512_setzero_si512(), scales[0], _mm512_packs_epi32(p1, p2)); ++ sumi = _mm512_dpwssd_epi32(sumi, scales[1], _mm512_packs_epi32(p3, p4)); ++ accd[iy] = _mm512_fmadd_ps(_mm512_set1_ps(deq.d*q8.scale(iy, i)), _mm512_cvtepi32_ps(sumi), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum256 = _mm256_add_ps(_mm512_castps512_ps256(accd[iy]), _mm512_extractf32x8_ps(accd[iy], 1)); ++ info.store(ix, iy, hsum_float_8(_mm256_add_ps(accm[iy], sum256))); ++ } ++ ++ } ++} ++ ++#else ++//====================================== AVX2 ================================================== ++ ++struct HighBit5 { ++ inline void load(const uint8_t * h) { hbits = _mm256_loadu_si256((const __m256i *)h); } ++ inline void apply(Q4Bits& bits, bool do_shift) { ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 3), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ if (do_shift) { ++ hbits = _mm256_srli_epi16(hbits, 4); ++ } ++ } ++ const __m256i mh = _mm256_set1_epi8(0x10); ++ __m256i hbits; ++}; ++ ++struct HighBit3 { ++ inline void load(const uint8_t * h) { hbits = _mm256_loadu_si256((const __m256i *)h); } ++ inline void apply(Q2Bits& bits, bool do_shift) { ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(hbits, mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_srli_epi16(hbits, 1), mh)); ++ if (do_shift) { ++ hbits = _mm256_srli_epi16(hbits, 4); ++ } ++ } ++ const __m256i mh = _mm256_set1_epi8(0x04); ++ __m256i hbits; ++}; ++ ++template ++inline void compute_block(int iy, int i, float d, const Q8& q8, const __m512i * values, const __m512i * scales, __m512 * accd) { ++ const __m512i p1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[0], q8.load_quants64(iy, i, 0)); ++ const __m512i p2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[1], q8.load_quants64(iy, i, 1)); ++ const __m512i p3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[2], q8.load_quants64(iy, i, 2)); ++ const __m512i p4 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[3], q8.load_quants64(iy, i, 3)); ++ auto sumi = _mm512_dpwssd_epi32(_mm512_setzero_si512(), scales[0], _mm512_packs_epi32(p1, p2)); ++ sumi = _mm512_dpwssd_epi32(sumi, scales[1], _mm512_packs_epi32(p3, p4)); ++ accd[iy] = _mm512_fmadd_ps(_mm512_set1_ps(d*q8.scale(iy, i)), _mm512_cvtepi32_ps(sumi), accd[iy]); ++} ++ ++struct DequantizerQ2K final : public BaseDequantizer { ++ DequantizerQ2K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m256i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales); ++ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4); ++ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); ++ process_mins_16(_mm256_cvtepi8_epi16(mins8), q8, i, -GGML_FP16_TO_FP32(x[i].data.data.dmin), accm); ++ prepare_scales_16(_mm256_cvtepi8_epi16(scales8), scales); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ } ++ ++ Q2Bits bits; ++ ++ const __m128i m4 = _mm_set1_epi8(0xf); ++}; ++ ++struct DequantizerQ3K final : public BaseDequantizer { ++ DequantizerQ3K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m256i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ hbits.load(x[i].hmask); ++ process_mins_and_scales_16(sc3.make_scales((const uint16_t *)x[i].scales), q8, i, -4.f*d, accm, scales); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ hbits.apply(bits, j == 0); ++ } ++ ++ Q2Bits bits; ++ HighBit3 hbits; ++ ScaleQ3 sc3; ++ ++ const __m128i m32 = _mm_set1_epi8(-32); ++}; ++ ++struct DequantizerQ4K final : public BaseDequantizer { ++ DequantizerQ4K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline __m256i new_block(int i, const Q8& q8, __m256 * accd) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ return s8k.process_mins_and_scales(x[i].scales, -GGML_FP16_TO_FP32(x[i].data.data.dmin), i, q8, accd); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ } ++ ++ Q4Bits bits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ5K final : public BaseDequantizer { ++ DequantizerQ5K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline __m256i new_block(int i, const Q8& q8, __m256 * accd) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ hbits.load(x[i].qh); ++ return s8k.process_mins_and_scales(x[i].scales, -GGML_FP16_TO_FP32(x[i].data.data.dmin), i, q8, accd); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ hbits.apply(bits, j == 0); ++ } ++ ++ Q4Bits bits; ++ HighBit5 hbits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ6K final : public BaseDequantizer { ++ DequantizerQ6K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m256i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ process_mins_and_scales_16(_mm_loadu_si128((const __m128i *)x[i].scales), q8, i, -32.f*d, accm, scales); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare64(x[i].ql, j); ++ auto hbits = _mm256_loadu_si256((const __m256i *)x[i].qh + j); ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(hbits, mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_srli_epi16(hbits, 2), mh)); ++ } ++ ++ Q4Bits bits; ++ const __m256i mh = _mm256_set1_epi8(0x30); ++}; ++ ++struct DequantizerIQ4XS final : public BaseDequantizer { ++ DequantizerIQ4XS(const void * vx, size_t bx) : BaseDequantizer(vx, bx), ++ values(MM256_SET1_M128I(_mm_loadu_si128((const __m128i *)kvalues_iq4nl))) {} ++ template ++ inline __m256i new_block(int i, [[maybe_unused]] const Q8& q8, [[maybe_unused]] __m256 * accd) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ auto scales128 = siq4.make_scales(*(const uint32_t *)x[i].scales_l, x[i].scales_h); ++ return MM256_SET1_M128I(scales128); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare16(x[i].qs, j); ++ bits.values[0] = _mm256_shuffle_epi8(values, bits.values[0]); ++ bits.values[1] = _mm256_shuffle_epi8(values, bits.values[1]); ++ bits.values[2] = _mm256_shuffle_epi8(values, bits.values[2]); ++ bits.values[3] = _mm256_shuffle_epi8(values, bits.values[3]); ++ } ++ ++ Q4Bits bits; ++ ScaleIQ4XS siq4; ++ const __m256i values; ++}; ++ ++template ++static void mul_mat_qX_K_q8_K_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ __m256 accd[nrc_y]; ++ __m256i scales[4]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ auto all_scales = deq.new_block(i, q8, accd); ++ ++ __m256i sumi[nrc_y]; ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ ++ deq.prepare(i, j); ++ ++ set_scales_8(all_scales, j, scales); ++ ++ if constexpr (std::is_same_v) { ++ multiply_add_avx2(deq.bits, scales, j, i, q8, sumi); ++ } else { ++ multiply_add(deq.bits, scales, j, i, q8, sumi); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ const __m256 vd = _mm256_set1_ps(deq.d*q8.scale(iy, i)); ++ accd[iy] = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi[iy]), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, hsum_float_8(accd[iy])); ++ } ++ ++ } ++} ++ ++template ++static void mul_mat_qY_K_q8_K_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n%QK_K == 0); ++ const int nb = n/QK_K; ++ ++ Q8 q8(info); ++ ++ __m256i all_scales[2]; ++ __m256i scales[4]; ++ __m256 accd[nrc_y]; ++ ++ Dequantizer deq(vx, bx); ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm256_setzero_ps(); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ deq.new_block(i, q8, accd, all_scales); ++ ++ __m256i sumi[nrc_y]; ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ deq.prepare(i, j); ++ set_scales_16(all_scales[j], scales); ++ multiply_add(deq.bits, scales, j, i, q8, sumi); ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ accd[iy] = _mm256_fmadd_ps(_mm256_set1_ps(deq.d*q8.scale(iy, i)), _mm256_cvtepi32_ps(sumi[iy]), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, hsum_float_8(accd[iy])); ++ } ++ ++ } ++ ++} ++ ++#endif ++ ++template void set_functions(mul_mat_t * funcs) { ++#ifdef HAVE_FANCY_SIMD ++ if constexpr (std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_iqX_k_q8_K_AVX512_dpbusd, Dequantizer, funcs) ++ } else { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_AVX512, Dequantizer, funcs) ++ funcs[0] = mul_mat_qX_K_q8_K_AVX512_1; ++ } ++#else ++ if constexpr (std::is_same_v || ++ std::is_same_v || ++ std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qY_K_q8_K_T, Dequantizer, funcs) ++ } else { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, Dequantizer, funcs) ++ } ++#endif ++} ++ ++} // namespace ++ ++bool iqk_set_kernels_kquants(int typeA, mul_mat_t * kernels) { ++ switch (typeA) { ++ case GGML_TYPE_Q2_K: set_functions(kernels); break; ++ case GGML_TYPE_Q3_K: set_functions(kernels); break; ++ case GGML_TYPE_Q4_K: set_functions(kernels); break; ++ case GGML_TYPE_Q5_K: set_functions(kernels); break; ++ case GGML_TYPE_Q6_K: set_functions(kernels); break; ++ case GGML_TYPE_IQ4_XS: set_functions(kernels); break; ++ default: return false; ++ } ++ return true; ++} ++ ++#else ++// --------------------------------- __aarch64__ -------------------------------------- ++ ++namespace { ++ ++template ++inline void accum_mins_8(const int16x8_t& mins, const Q8& q8, float32x4_t * acc, int i, float c) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto q8s = q8.load_bsums8(iy, i); ++ int32x4_t b1 = vmull_s16(vget_low_s16(mins), vget_low_s16(q8s)); ++ int32x4_t b2 = vmull_s16(vget_high_s16(mins), vget_high_s16(q8s)); ++ float32x4_t prod = vcvtq_f32_s32(vaddq_s32(b1, b2)); ++ acc[iy] = vmlaq_f32(acc[iy], prod, vdupq_n_f32(c*q8.scale(iy, i))); ++ } ++} ++template ++inline void accum_mins_16(const int16x8x2_t& mins, const Q8& q8, float32x4_t * acc, int i, float c) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto q8s = q8.load_bsums(iy, i); ++ int32x4_t b1 = vmull_s16(vget_low_s16 (mins.val[0]), vget_low_s16 (q8s.val[0])); ++ int32x4_t b2 = vmull_s16(vget_high_s16(mins.val[0]), vget_high_s16(q8s.val[0])); ++ int32x4_t b3 = vmull_s16(vget_low_s16 (mins.val[1]), vget_low_s16 (q8s.val[1])); ++ int32x4_t b4 = vmull_s16(vget_high_s16(mins.val[1]), vget_high_s16(q8s.val[1])); ++ float32x4_t prod = vcvtq_f32_s32(vaddq_s32(vaddq_s32(b1, b2), vaddq_s32(b3, b4))); ++ acc[iy] = vmlaq_f32(acc[iy], prod, vdupq_n_f32(c*q8.scale(iy, i))); ++ } ++} ++ ++struct Scales8 { ++ uint32_t utmp[4]; ++ const uint8_t * sc8 = (const uint8_t *)utmp; ++ template ++ inline int32x4x2_t process_scales_mins(const Qx& x, const Q8& q8, int i, float32x4_t * acc) { ++ make_q4_scales(x.scales, utmp); ++ int16x8_t mins = vmovl_s8(vld1_s8((const int8_t *)sc8 + 8)); ++ accum_mins_8(mins, q8, acc, i, -GGML_FP16_TO_FP32(x.data.data.dmin)); ++ ++ uint8x8_t scales8 = vld1_u8(sc8); ++ uint16x8_t scales16 = vmovl_u8(scales8); ++ int32x4x2_t scales = {vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales16))), ++ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales16)))}; ++ return scales; ++ } ++ inline float32x4x4_t make_scales(float d, float m, const uint8_t * scales) { ++ make_q4_scales(scales, utmp); ++ auto d16 = vmovl_u8(vld1_u8(sc8+0)); ++ auto m16 = vmovl_u8(vld1_u8(sc8+8)); ++ auto vd = vdupq_n_f32(d); ++ auto vm = vdupq_n_f32(m); ++ return { vmulq_f32(vd, vcvtq_f32_u32(vmovl_u16(vget_low_u16 (d16)))), ++ vmulq_f32(vd, vcvtq_f32_u32(vmovl_u16(vget_high_u16(d16)))), ++ vmulq_f32(vm, vcvtq_f32_u32(vmovl_u16(vget_low_u16 (m16)))), ++ vmulq_f32(vm, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m16)))) }; ++ } ++}; ++ ++struct DequantizerQ4K final : public BaseDequantizer { ++ DequantizerQ4K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ return s8.process_scales_mins(x[i], q8, i, acc); ++ } ++ inline float32x4x4_t new_block(int i) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ float m = -GGML_FP16_TO_FP32(x[i].data.data.dmin); ++ return s8.make_scales(d, m, x[i].scales); ++ } ++ inline void prepare(int i, int j) { ++ if (nrc == 1) bits.prepare_v2(x[i].qs+64*j); ++ else bits.prepare(x[i].qs+64*j); ++ } ++ ++ Q4bits bits; ++ Scales8 s8; ++ ++}; ++ ++struct HighBit5 { ++ const uint8x16_t mhb = vdupq_n_u8(0x10); ++ uint8x16x2_t bits; ++ inline void apply(uint8x16x4_t& b1, uint8x16x4_t& b2, bool do_shift) { ++ b1.val[0] = vorrq_u8(b1.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 4), mhb)); ++ b1.val[1] = vorrq_u8(b1.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 4), mhb)); ++ b1.val[2] = vorrq_u8(b1.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 3), mhb)); ++ b1.val[3] = vorrq_u8(b1.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 3), mhb)); ++ ++ b2.val[0] = vorrq_u8(b2.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 2), mhb)); ++ b2.val[1] = vorrq_u8(b2.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 2), mhb)); ++ b2.val[2] = vorrq_u8(b2.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 1), mhb)); ++ b2.val[3] = vorrq_u8(b2.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 1), mhb)); ++ ++ if (do_shift) { ++ bits.val[0] = vshrq_n_u8(bits.val[0], 4); ++ bits.val[1] = vshrq_n_u8(bits.val[1], 4); ++ } ++ } ++}; ++ ++struct HighBit3 { ++ const uint8x16_t mhb = vdupq_n_u8(0x04); ++ uint8x16x2_t bits; ++ inline void apply(uint8x16x4_t& b1, uint8x16x4_t& b2, bool do_shift) { ++ b1.val[0] = vorrq_u8(b1.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 2), mhb)); ++ b1.val[1] = vorrq_u8(b1.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 2), mhb)); ++ b1.val[2] = vorrq_u8(b1.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 1), mhb)); ++ b1.val[3] = vorrq_u8(b1.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 1), mhb)); ++ ++ b2.val[0] = vorrq_u8(b2.val[0], vandq_u8(bits.val[0], mhb)); ++ b2.val[1] = vorrq_u8(b2.val[1], vandq_u8(bits.val[1], mhb)); ++ b2.val[2] = vorrq_u8(b2.val[2], vandq_u8(vshrq_n_u8(bits.val[0], 1), mhb)); ++ b2.val[3] = vorrq_u8(b2.val[3], vandq_u8(vshrq_n_u8(bits.val[1], 1), mhb)); ++ ++ if (do_shift) { ++ bits.val[0] = vshrq_n_u8(bits.val[0], 4); ++ bits.val[1] = vshrq_n_u8(bits.val[1], 4); ++ } ++ } ++}; ++ ++struct DequantizerQ5K final : public BaseDequantizer { ++ DequantizerQ5K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ h.bits = vld1q_u8_x2(x[i].qh); ++ return s8.process_scales_mins(x[i], q8, i, acc); ++ } ++ inline float32x4x4_t new_block(int i) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ float m = -GGML_FP16_TO_FP32(x[i].data.data.dmin); ++ h.bits = vld1q_u8_x2(x[i].qh); ++ return s8.make_scales(d, m, x[i].scales); ++ } ++ inline void prepare(int i, int j) { ++ if (nrc == 1) bits.prepare_v2(x[i].qs+64*j); ++ else bits.prepare(x[i].qs+64*j); ++ h.apply(bits.b1, bits.b2, j == 0); ++ } ++ ++ Q4bits bits; ++ HighBit5 h; ++ Scales8 s8; ++ ++ uint8x16x2_t hbits; ++ ++}; ++ ++inline int32x4x4_t make_wider(const int16x8x2_t& scales16) { ++ int32x4x4_t scales = { ++ vmovl_s16(vget_low_s16 (scales16.val[0])), ++ vmovl_s16(vget_high_s16(scales16.val[0])), ++ vmovl_s16(vget_low_s16 (scales16.val[1])), ++ vmovl_s16(vget_high_s16(scales16.val[1])), ++ }; ++ return scales; ++} ++ ++template ++inline int32x4x4_t process_scales_mins_16(const int8x16_t& scales8, const Q8& q8, float32x4_t * acc, int i, float c) { ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(scales8)); ++ scales16.val[1] = vmovl_s8(vget_high_s8(scales8)); ++ accum_mins_16(scales16, q8, acc, i, c); ++ return make_wider(scales16); ++} ++ ++struct DequantizerQ6K final : public BaseDequantizer { ++ DequantizerQ6K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ return process_scales_mins_16(vld1q_s8(x[i].scales), q8, acc, i, -32.f*d); ++ } ++ inline float32x4x4_t new_block(int i) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ auto vd = vdupq_n_f32(d); ++ auto scales8 = vld1q_s8(x[i].scales); ++ auto scales16_1 = vmovl_s8(vget_low_s8 (scales8)); ++ auto scales16_2 = vmovl_s8(vget_high_s8(scales8)); ++ return { vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (scales16_1)))), ++ vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_high_s16(scales16_1)))), ++ vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (scales16_2)))), ++ vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_high_s16(scales16_2)))) }; ++ } ++ ++ inline void prepare(int i, int j) { ++ ++ auto hbits = vld1q_u8_x2(x[i].qh + 32*j); ++ ++ bits.prepare64(x[i].ql+64*j); ++ bits.b1.val[0] = vorrq_u8(bits.b1.val[0], vandq_u8(vshlq_n_u8(hbits.val[0], 4), mhb)); ++ bits.b1.val[1] = vorrq_u8(bits.b1.val[1], vandq_u8(vshlq_n_u8(hbits.val[1], 4), mhb)); ++ bits.b1.val[2] = vorrq_u8(bits.b1.val[2], vandq_u8(vshlq_n_u8(hbits.val[0], 2), mhb)); ++ bits.b1.val[3] = vorrq_u8(bits.b1.val[3], vandq_u8(vshlq_n_u8(hbits.val[1], 2), mhb)); ++ ++ bits.b2.val[0] = vorrq_u8(bits.b2.val[0], vandq_u8(hbits.val[0], mhb)); ++ bits.b2.val[1] = vorrq_u8(bits.b2.val[1], vandq_u8(hbits.val[1], mhb)); ++ bits.b2.val[2] = vorrq_u8(bits.b2.val[2], vandq_u8(vshrq_n_u8(hbits.val[0], 2), mhb)); ++ bits.b2.val[3] = vorrq_u8(bits.b2.val[3], vandq_u8(vshrq_n_u8(hbits.val[1], 2), mhb)); ++ ++ } ++ ++ inline void prepare_signed(int i, int j) { ++ prepare(i, j); ++ auto m32 = vdupq_n_s8(-32); ++ for (int k = 0; k < 4; ++k) { ++ bits.b1.val[k] = vaddq_s8(bits.b1.val[k], m32); ++ bits.b2.val[k] = vaddq_s8(bits.b2.val[k], m32); ++ } ++ } ++ ++ Q4bits bits; ++ ++ const uint8x16_t mhb = vdupq_n_u8(0x30); ++ ++}; ++ ++struct DequantizerQ3K final : public BaseDequantizer { ++ DequantizerQ3K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ h.bits = vld1q_u8_x2(x[i].hmask); ++ mask = vdupq_n_u8(0x01); ++ const uint16_t * sc16 = (const uint16_t *)x[i].scales; ++ uint32_t aux0 = sc16[0] | (sc16[1] << 16); ++ uint32_t aux1 = sc16[2] | (sc16[3] << 16); ++ uint32_t aux2 = sc16[4] | (sc16[5] << 16); ++ aux32[0] = (aux0 & 0x0f0f0f0f) | ((aux2 << 4) & 0x30303030); ++ aux32[1] = (aux1 & 0x0f0f0f0f) | ((aux2 << 2) & 0x30303030); ++ aux32[2] = ((aux0 >> 4) & 0x0f0f0f0f) | ((aux2 >> 0) & 0x30303030); ++ aux32[3] = ((aux1 >> 4) & 0x0f0f0f0f) | ((aux2 >> 2) & 0x30303030); ++ auto scales8 = vaddq_s8(vld1q_s8((const int8_t *)aux32), vdupq_n_s8(-32)); ++ if (nrc > 1) { ++ return process_scales_mins_16(scales8, q8, acc, i, -4.f*d); ++ } ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(scales8)); ++ scales16.val[1] = vmovl_s8(vget_high_s8(scales8)); ++ return make_wider(scales16); ++ } ++ ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs+32*j); ++ if (nrc > 1) { ++ h.apply(bits.b1, bits.b2, j == 0); ++ } else { ++ auto minus4 = vdupq_n_u8(0xfc); ++ auto zero = vdupq_n_u8(0); ++ bits.b1.val[0] = vorrq_u8(bits.b1.val[0], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b1.val[1] = vorrq_u8(bits.b1.val[1], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ bits.b1.val[2] = vorrq_u8(bits.b1.val[2], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b1.val[3] = vorrq_u8(bits.b1.val[3], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ bits.b2.val[0] = vorrq_u8(bits.b2.val[0], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b2.val[1] = vorrq_u8(bits.b2.val[1], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ bits.b2.val[2] = vorrq_u8(bits.b2.val[2], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b2.val[3] = vorrq_u8(bits.b2.val[3], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ } ++ } ++ ++ uint32_t aux32[4]; ++ ++ Q2bits bits; ++ ++ uint8x16_t mask; ++ HighBit3 h; ++ ++}; ++ ++struct DequantizerQ2K final : public BaseDequantizer { ++ DequantizerQ2K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return true; } ++ ++ template ++ inline void process_scales(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].data.data.d); ++ auto scales_and_mins = vld1q_u8(x[i].scales); ++ auto mins8 = vreinterpretq_s8_u8(vshrq_n_u8(scales_and_mins, 4)); ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(mins8)); ++ scales16.val[1] = vmovl_s8(vget_high_s8(mins8)); ++ accum_mins_16(scales16, q8, acc, i, -GGML_FP16_TO_FP32(x[i].data.data.dmin)); ++ ++ scales8 = vandq_u8(scales_and_mins, vdupq_n_u8(0xf)); ++ } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ process_scales(i, q8, acc); ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(vreinterpretq_s8_u8(scales8))); ++ scales16.val[1] = vmovl_s8(vget_high_s8(vreinterpretq_s8_u8(scales8))); ++ return make_wider(scales16); ++ } ++ ++ template ++ inline void compute(const Q8& q8, int i, int j, int32x4_t * sumi) { ++ auto m1 = vdupq_n_u8(1); ++ auto shuffle = vdupq_n_u8(8*j); ++ bits.b1.val[0] = vmulq_u8(bits.b1.val[0], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b1.val[1] = vmulq_u8(bits.b1.val[1], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b1.val[2] = vmulq_u8(bits.b1.val[2], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b1.val[3] = vmulq_u8(bits.b1.val[3], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[0] = vmulq_u8(bits.b2.val[0], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[1] = vmulq_u8(bits.b2.val[1], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[2] = vmulq_u8(bits.b2.val[2], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[3] = vmulq_u8(bits.b2.val[3], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto q8b_1 = q8.load_quants(iy, i, 4*j+0); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b1.val[0]), q8b_1.val[0]), ++ vreinterpretq_s8_u8(bits.b1.val[1]), q8b_1.val[1]); ++ ++ auto q8b_2 = q8.load_quants(iy, i, 4*j+1); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b1.val[2]), q8b_2.val[0]), ++ vreinterpretq_s8_u8(bits.b1.val[3]), q8b_2.val[1]); ++ ++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b2.val[0]), q8b_3.val[0]), ++ vreinterpretq_s8_u8(bits.b2.val[1]), q8b_3.val[1]); ++ ++ auto q8b_4 = q8.load_quants(iy, i, 4*j+3); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b2.val[2]), q8b_4.val[0]), ++ vreinterpretq_s8_u8(bits.b2.val[3]), q8b_4.val[1]); ++ } ++ } ++ ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs+32*j); ++ } ++ ++ uint32_t aux32[4]; ++ ++ uint8x16_t scales8; ++ ++ Q2bits bits; ++ ++}; ++ ++struct DequantizerIQ4XS final : public BaseDequantizer { ++ ++ static int8x16_t load_values() { ++ static const int8_t iq4nl_values[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113}; ++ return vld1q_s8(iq4nl_values); ++ } ++ ++ DequantizerIQ4XS(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc), values(load_values()) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ inline void new_row(int ix) { x = (const block_iq4_xs *)((const char *)vx + bx*ix); } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ (void)q8; ++ (void)acc; ++ d = GGML_FP16_TO_FP32(x[i].d); ++ const uint16_t scales_h = x[i].scales_h; ++ const uint16_t * scales_l = (const uint16_t *)x[i].scales_l; ++ aux32[0] = scales_l[0] | (scales_l[1] << 16); ++ aux32[1] = aux32[0] >> 4; ++ // scl is ordered as 0, 2, 4, 6, 1, 3, 5, 7 ++ uint8x8_t scl8 = vand_u8(vld1_u8((const uint8_t *)aux32), vdup_n_u8(0xf)); ++ uint16_t * aux16 = (uint16_t *)aux32; ++ aux16[0] = scales_h << 4; aux16[1] = scales_h << 2; aux16[2] = scales_h; aux16[3] = scales_h >> 2; ++ // sch is ordered as 0, 4, 1, 5, 2, 6, 3, 7 ++ uint8x8_t sch8 = vand_u8(vld1_u8((const uint8_t *)aux16), vdup_n_u8(0x30)); ++ int8x8_t scales8 = vadd_s8(vreinterpret_s8_u8(vorr_u8(scl8, vtbl1_u8(sch8, vreinterpret_u8_u32(hshuff)))), vdup_n_s8(-32)); ++ // shuffle 0, 2, 4, 6, 1, 3, 5, 7 -> 0, 1, 2, 3, 4, 5, 6, 7 ++ scales8 = vtbl1_s8(scales8, vreinterpret_s8_u32(hshuff)); ++ int16x8_t scales16 = vmovl_s8(scales8); ++ int32x4x2_t scales = {vmovl_s16(vget_low_s16(scales16)), vmovl_s16(vget_high_s16(scales16))}; ++ return scales; ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare16(x[i].qs+64*j); ++ //if (nrc == 1) { ++ // bits.prepare16_v2(x[i].qs+64*j); ++ //} else { ++ // bits.prepare16(x[i].qs+64*j); ++ //} ++ for (int k = 0; k < 4; ++k) { ++ bits.b1.val[k] = vreinterpretq_u8_s8(vqtbl1q_s8(values, bits.b1.val[k])); ++ bits.b2.val[k] = vreinterpretq_u8_s8(vqtbl1q_s8(values, bits.b2.val[k])); ++ } ++ } ++ ++ Q4bits bits; ++ const int8x16_t values; ++ uint32_t aux32[2]; ++ ++ constexpr static uint32x2_t hshuff = {0x05010400, 0x07030602}; ++ ++}; ++ ++} // namespace ++ ++bool iqk_set_kernels_kquants(int typeA, mul_mat_t * kernels) { ++ switch (typeA) { ++ case GGML_TYPE_Q2_K: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerQ2K, kernels); break; ++ case GGML_TYPE_Q3_K: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerQ3K, kernels); break; ++ case GGML_TYPE_Q4_K: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerQ4K, kernels); break; ++ case GGML_TYPE_Q5_K: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerQ5K, kernels); break; ++ case GGML_TYPE_Q6_K: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerQ6K, kernels); break; ++ case GGML_TYPE_IQ4_XS: IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerIQ4XS, kernels); break; ++ default: return false; ++ } ++ return true; ++} ++ ++#endif ++ ++#endif // IQK_IMPLEMENT From caaa6d048f8585c2b9e8278893d09af574204487 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 4 Nov 2026 22:42:38 -0800 Subject: [PATCH 2/5] ggml-patches/0018: explicit NEON reinterprets so GCC builds the aarch64 kernels (no lax vector conversions) --- .../ggml-patches/0018-ggml-cpu-iqk-gemm.patch | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch index fa138d843..003e60e00 100644 --- a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch +++ b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch @@ -1142,7 +1142,7 @@ index 00000000..466e942e +#endif diff --git a/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp b/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp new file mode 100644 -index 00000000..f9596cfb +index 00000000..43ba7450 --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp @@ -0,0 +1,364 @@ @@ -1377,7 +1377,7 @@ index 00000000..f9596cfb + scales128 = vaddq_u16(vshlq_n_u16(scales128, 1), vdupq_n_u16(1)); + auto mask = vceqq_u16(vandq_u16(qhb, delta_mask), delta_mask); + // Note: we explicitly assume IQ1S_DELTA = 0.125 -+ auto deltas128 = vsubq_s16(vbicq_s16(scales128, mask), vandq_s16(scales128, mask)); ++ auto deltas128 = vsubq_s16(vreinterpretq_s16_u16(vbicq_u16(scales128, mask)), vreinterpretq_s16_u16(vandq_u16(scales128, mask))); + //auto deltas128 = vorrq_s16(vandq_s16(vdupq_n_s16(-1), mask), vbicq_s16(vdupq_n_s16(1), mask)); + //deltas128 = vmulq_s16(scales128, deltas128); + scales128 = vshlq_n_u16(scales128, 3); @@ -1443,14 +1443,14 @@ index 00000000..f9596cfb + auto aux8 = vld1_u8(iq1m[ibl].scales); + auto aux16 = vcombine_u8(aux8, aux8); + uint16x8x2_t sc16 = { vreinterpretq_u16_u8(vqtbl1q_u8(aux16, scale_shuffle.val[0])), vreinterpretq_u16_u8(vqtbl1q_u8(aux16, scale_shuffle.val[1])) }; -+ sc16.val[0] = vmulq_u16(vandq_u16(sc16.val[0], vdupq_n_u64(0x0e0001c000380007)), vdupq_n_u64(0x0001000800400200)); -+ sc16.val[1] = vmulq_u16(vandq_u16(sc16.val[1], vdupq_n_u64(0x0e0001c000380007)), vdupq_n_u64(0x0001000800400200)); ++ sc16.val[0] = vmulq_u16(vandq_u16(sc16.val[0], vreinterpretq_u16_u64(vdupq_n_u64(0x0e0001c000380007))), vreinterpretq_u16_u64(vdupq_n_u64(0x0001000800400200))); ++ sc16.val[1] = vmulq_u16(vandq_u16(sc16.val[1], vreinterpretq_u16_u64(vdupq_n_u64(0x0e0001c000380007))), vreinterpretq_u16_u64(vdupq_n_u64(0x0001000800400200))); + sc16.val[0] = vaddq_u16(vshrq_n_u16(sc16.val[0], 8), vdupq_n_u16(1)); + sc16.val[1] = vaddq_u16(vshrq_n_u16(sc16.val[1], 8), vdupq_n_u16(1)); -+ scales.val[0] = vmovl_s16(vget_low_s16 (sc16.val[0])); -+ scales.val[1] = vmovl_s16(vget_high_s16(sc16.val[0])); -+ scales.val[2] = vmovl_s16(vget_low_s16 (sc16.val[1])); -+ scales.val[3] = vmovl_s16(vget_high_s16(sc16.val[1])); ++ scales.val[0] = vmovl_s16(vget_low_s16 (vreinterpretq_s16_u16(sc16.val[0]))); ++ scales.val[1] = vmovl_s16(vget_high_s16(vreinterpretq_s16_u16(sc16.val[0]))); ++ scales.val[2] = vmovl_s16(vget_low_s16 (vreinterpretq_s16_u16(sc16.val[1]))); ++ scales.val[3] = vmovl_s16(vget_high_s16(vreinterpretq_s16_u16(sc16.val[1]))); + for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { + qx[2*ib64+0] = {vreinterpretq_s8_u64(uint64x2_t{iq1s_grid_us[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)], + iq1s_grid_us[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)]}), @@ -1512,7 +1512,7 @@ index 00000000..f9596cfb +#endif // IQK_IMPLEMENT diff --git a/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp b/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp new file mode 100644 -index 00000000..72027c76 +index 00000000..f916fee0 --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp @@ -0,0 +1,1169 @@ @@ -2633,7 +2633,7 @@ index 00000000..72027c76 + } + + static inline void make2(SignHelper& sh, const uint8x16_t& signs16, const uint16x8_t& idx_l, uint8_t qh, -+ const int8x16_t& hshift, uint8x16_t * b) { ++ const int16x8_t& hshift, uint8x16_t * b) { + auto vindex = vorrq_u16(idx_l, vandq_u16(vshlq_u16(vdupq_n_u16(qh), hshift), vdupq_n_u16(256))); + const uint16_t * idx = (const uint16_t *)&vindex; + b[0] = vreinterpretq_u8_u32(uint32x4_t{iq3s_grid[idx[0]], iq3s_grid[idx[1]], iq3s_grid[idx[2]], iq3s_grid[idx[3]]}); @@ -2642,7 +2642,7 @@ index 00000000..72027c76 + sh.apply_signs_1(b+1, signs16); + } + static inline void make4(SignHelper& sh, const uint8x16_t& signs16, const uint8_t * qs, const uint8_t * qh, -+ const int8x16_t& hshift, uint8x16_t * b) { ++ const int16x8_t& hshift, uint8x16_t * b) { + auto idx_l = vld1q_u8(qs); + make2(sh, signs16, vmovl_u8(vget_low_u8 (idx_l)), qh[0], hshift, b+0); + make2(sh, signs16, vmovl_u8(vget_high_u8(idx_l)), qh[1], hshift, b+2); @@ -2687,7 +2687,7 @@ index 00000000..72027c76 +#endif // IQK_IMPLEMENT diff --git a/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp b/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp new file mode 100644 -index 00000000..2dd6980a +index 00000000..916de2a3 --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp @@ -0,0 +1,1162 @@ @@ -3636,8 +3636,8 @@ index 00000000..2dd6980a + prepare(i, j); + auto m32 = vdupq_n_s8(-32); + for (int k = 0; k < 4; ++k) { -+ bits.b1.val[k] = vaddq_s8(bits.b1.val[k], m32); -+ bits.b2.val[k] = vaddq_s8(bits.b2.val[k], m32); ++ bits.b1.val[k] = vreinterpretq_u8_s8(vaddq_s8(vreinterpretq_s8_u8(bits.b1.val[k]), m32)); ++ bits.b2.val[k] = vreinterpretq_u8_s8(vaddq_s8(vreinterpretq_s8_u8(bits.b2.val[k]), m32)); + } + } + From 0ad6060ce4eb65a3ea73856b1d4f0b4eca07ede0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 4 Nov 2026 23:06:05 -0800 Subject: [PATCH 3/5] ggml-patches/0018: iqk kernels default off on x86 (the tiled K-quant path measured faster), on for ARM --- .../ggml-patches/0018-ggml-cpu-iqk-gemm.patch | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch index 003e60e00..d5b9be9f7 100644 --- a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch +++ b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch @@ -16,7 +16,7 @@ index d347757e..9bcd6009 100644 ggml-cpu/common.h ggml-cpu/binary-ops.h diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c -index c10f0679..446a361e 100644 +index c10f0679..d8054cab 100644 --- a/src/ggml-cpu/ggml-cpu.c +++ b/src/ggml-cpu/ggml-cpu.c @@ -15,6 +15,7 @@ @@ -32,7 +32,7 @@ index c10f0679..446a361e 100644 } + // ggml_cpu_iqk_mul_mat (ik_llama.cpp kernels) runs below for its weight types once src1 is in Q8_K, in place of -+ // vec_dot. The x86 tiled K-quant path still goes first (it takes ne11 >= 8) unless GGML_CPU_IQK=2. ++ // vec_dot (on by default on ARM only). The x86 tiled K-quant path still goes first unless GGML_CPU_IQK=2. + const bool iqk = !params->use_ref && ggml_cpu_iqk_type_enabled(src0->type) && + type_traits_cpu[src0->type].vec_dot_type == GGML_TYPE_Q8_K && dst->type == GGML_TYPE_F32 && + src0->ne[0] % 256 == 0; @@ -866,10 +866,10 @@ index 00000000..478cf1a4 +#endif // IQK_IMPLEMENT diff --git a/src/ggml-cpu/iqk/iqk_gemm.cpp b/src/ggml-cpu/iqk/iqk_gemm.cpp new file mode 100644 -index 00000000..f82a1c5e +index 00000000..addc939b --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm.cpp -@@ -0,0 +1,224 @@ +@@ -0,0 +1,231 @@ +// +// Copyright (C) 2024-2025 Iwan Kawrakow +// MIT license @@ -995,6 +995,13 @@ index 00000000..f82a1c5e +void ggml_cpu_iqk_init(void) { + uint64_t mask = k_iqk_built_types; + const char * on = getenv("GGML_CPU_IQK"); ++#if defined(__x86_64__) || defined(_M_X64) ++ // x86: off unless asked for. The tiled K-quant path takes ne11 >= 8 and measured faster than these kernels on ++ // Zen 3/4/5, Broadwell and Emerald Rapids, so turning them on by default would only change ne11 < 8 results. ++ if (on == NULL) { ++ mask = 0; ++ } ++#endif + if (on != NULL && atoi(on) == 0) { + mask = 0; + } @@ -1096,10 +1103,10 @@ index 00000000..f82a1c5e +} // extern "C" diff --git a/src/ggml-cpu/iqk/iqk_gemm.h b/src/ggml-cpu/iqk/iqk_gemm.h new file mode 100644 -index 00000000..466e942e +index 00000000..3cf83d93 --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm.h -@@ -0,0 +1,40 @@ +@@ -0,0 +1,41 @@ +#pragma once + +// GEMM kernels ported from ik_llama.cpp (iqk_gemm.cpp) for the k-quant / i-quant weight types whose @@ -1114,7 +1121,8 @@ index 00000000..466e942e +extern "C" { +#endif + -+// Reads once: GGML_CPU_IQK (0 disables; 2 runs these kernels ahead of the x86 tiled K-quant path instead of after it), ++// Reads once: GGML_CPU_IQK (0 disables; 1 enables, the default on ARM, while x86 defaults to off; 2 also runs them ++// ahead of the x86 tiled K-quant path), +// GGML_CPU_IQK_TYPES (ggml type names or "all"; the other types keep the tiled / vec_dot paths) and GGML_CPU_IQK_DEBUG +// (1: print the enabled types and the first mul_mat per type to stderr). +void ggml_cpu_iqk_init(void); From ea0904db8880648151e7df42765b45dad2ccca77 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 5 Nov 2026 07:57:16 -0800 Subject: [PATCH 4/5] ggml-patches/0018: iqk kernels by batch size on x86 by default test-backend-ops MUL_MAT perf (m 4096, k 14336) on Zen 3, Zen 4, Zen 5, Xeon 8370C / 8573C and Core i7-8700B: iqk is 1.07-3.2x faster than vec_dot for 2-7 src1 rows and 1.15-2.9x faster than the tiled path at 8 rows (not iq2_xxs / iq3_s, 0.84x / 0.91x on Zen 5), and 1.01-2.3x for K-quants up to 12 rows; tiled wins from 16 rows up and vec_dot at 1 row. x86 now takes iqk exactly there when GGML_CPU_IQK is unset; GGML_CPU_IQK=0 / 1 / 2 keep their meaning. --- .../ggml-patches/0018-ggml-cpu-iqk-gemm.patch | 86 +++++++++++++------ 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch index d5b9be9f7..4f408e4df 100644 --- a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch +++ b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch @@ -1,5 +1,5 @@ diff --git a/src/ggml-cpu/CMakeLists.txt b/src/ggml-cpu/CMakeLists.txt -index d347757e..9bcd6009 100644 +index d347757..9bcd600 100644 --- a/src/ggml-cpu/CMakeLists.txt +++ b/src/ggml-cpu/CMakeLists.txt @@ -45,6 +45,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) @@ -16,7 +16,7 @@ index d347757e..9bcd6009 100644 ggml-cpu/common.h ggml-cpu/binary-ops.h diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c -index c10f0679..d8054cab 100644 +index 6f9319d..6b6ca1c 100644 --- a/src/ggml-cpu/ggml-cpu.c +++ b/src/ggml-cpu/ggml-cpu.c @@ -15,6 +15,7 @@ @@ -27,23 +27,26 @@ index c10f0679..d8054cab 100644 #if defined(_MSC_VER) || defined(__MINGW32__) #include // using malloc.h with MSC/MINGW -@@ -1455,8 +1456,14 @@ void ggml_compute_forward_mul_mat( +@@ -1529,8 +1530,17 @@ void ggml_compute_forward_mul_mat( return; } + // ggml_cpu_iqk_mul_mat (ik_llama.cpp kernels) runs below for its weight types once src1 is in Q8_K, in place of -+ // vec_dot (on by default on ARM only). The x86 tiled K-quant path still goes first unless GGML_CPU_IQK=2. -+ const bool iqk = !params->use_ref && ggml_cpu_iqk_type_enabled(src0->type) && -+ type_traits_cpu[src0->type].vec_dot_type == GGML_TYPE_Q8_K && dst->type == GGML_TYPE_F32 && -+ src0->ne[0] % 256 == 0; ++ // vec_dot: every batch size on ARM, small batches on x86 (ggml_cpu_iqk_mode), where they also go ahead of the ++ // tiled K-quant path. ++ const int iqk_mode = !params->use_ref && type_traits_cpu[src0->type].vec_dot_type == GGML_TYPE_Q8_K && ++ dst->type == GGML_TYPE_F32 && src0->ne[0] % 256 == 0 ++ ? ggml_cpu_iqk_mode(src0->type, src1->ne[1]) ++ : 0; ++ const bool iqk = iqk_mode != 0; + // If tiled is supported, it will execute the full op here and we return - if (ggml_compute_forward_mul_mat_tiled(params, dst)) { -+ if (!(iqk && ggml_cpu_iqk_before_tiled()) && ggml_compute_forward_mul_mat_tiled(params, dst)) { ++ if (iqk_mode != 2 && ggml_compute_forward_mul_mat_tiled(params, dst)) { return; } -@@ -1597,6 +1604,26 @@ UseGgmlGemm1:; +@@ -1671,6 +1681,26 @@ UseGgmlGemm1:; UseGgmlGemm2:; #endif @@ -70,7 +73,7 @@ index c10f0679..d8054cab 100644 // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) const int64_t nr0 = ne0; -@@ -4126,6 +4153,7 @@ void ggml_cpu_init(void) { +@@ -4200,6 +4230,7 @@ void ggml_cpu_init(void) { #ifdef GGML_USE_LLAMAFILE ggml_cpu_sgemm_types_init(); #endif @@ -79,7 +82,7 @@ index c10f0679..d8054cab 100644 GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0); diff --git a/src/ggml-cpu/ggml-cpu.cpp b/src/ggml-cpu/ggml-cpu.cpp -index 9cef95e1..31de8537 100644 +index 9cef95e..31de853 100644 --- a/src/ggml-cpu/ggml-cpu.cpp +++ b/src/ggml-cpu/ggml-cpu.cpp @@ -5,6 +5,7 @@ @@ -106,7 +109,7 @@ index 9cef95e1..31de8537 100644 if (strcmp(name, "ggml_threadpool_new") == 0) { diff --git a/src/ggml-cpu/iqk/iqk_common.h b/src/ggml-cpu/iqk/iqk_common.h new file mode 100644 -index 00000000..478cf1a4 +index 0000000..478cf1a --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_common.h @@ -0,0 +1,754 @@ @@ -866,10 +869,10 @@ index 00000000..478cf1a4 +#endif // IQK_IMPLEMENT diff --git a/src/ggml-cpu/iqk/iqk_gemm.cpp b/src/ggml-cpu/iqk/iqk_gemm.cpp new file mode 100644 -index 00000000..addc939b +index 0000000..1899fd7 --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm.cpp -@@ -0,0 +1,231 @@ +@@ -0,0 +1,258 @@ +// +// Copyright (C) 2024-2025 Iwan Kawrakow +// MIT license @@ -930,6 +933,7 @@ index 00000000..addc939b +std::atomic g_iqk_count[64]; +int g_iqk_debug = 0; +bool g_iqk_first = false; ++bool g_iqk_auto = false; + +#ifdef IQK_IMPLEMENT + @@ -996,11 +1000,9 @@ index 00000000..addc939b + uint64_t mask = k_iqk_built_types; + const char * on = getenv("GGML_CPU_IQK"); +#if defined(__x86_64__) || defined(_M_X64) -+ // x86: off unless asked for. The tiled K-quant path takes ne11 >= 8 and measured faster than these kernels on -+ // Zen 3/4/5, Broadwell and Emerald Rapids, so turning them on by default would only change ne11 < 8 results. -+ if (on == NULL) { -+ mask = 0; -+ } ++ // x86: by default the kernels take only the small batches where they measured faster (ggml_cpu_iqk_mode); the ++ // tiled K-quant path keeps the large ones, where it measured 1.2-5x faster. ++ g_iqk_auto = on == NULL; +#endif + if (on != NULL && atoi(on) == 0) { + mask = 0; @@ -1026,7 +1028,7 @@ index 00000000..addc939b + g_iqk_debug = dbg != NULL ? atoi(dbg) : 0; + if (g_iqk_debug) { + fprintf(stderr, "ggml-cpu iqk gemm: built types 0x%llx, enabled 0x%llx (%s the tiled path):", (unsigned long long) k_iqk_built_types, -+ (unsigned long long) mask, g_iqk_first ? "ahead of" : "after"); ++ (unsigned long long) mask, g_iqk_auto ? "by batch size around" : g_iqk_first ? "ahead of" : "after"); + for (int t = 0; t < GGML_TYPE_COUNT && t < 64; ++t) { + if ((mask >> t) & 1) fprintf(stderr, " %s", ggml_type_name((enum ggml_type) t)); + } @@ -1042,6 +1044,34 @@ index 00000000..addc939b + return g_iqk_first; +} + ++// x86 default (GGML_CPU_IQK unset), from test-backend-ops MUL_MAT perf (m = 4096, k = 14336) on Zen 3 (EPYC 7763), ++// Zen 4 (EPYC 9V74), Zen 5 (EPYC 9V45), Xeon 8370C / 8573C and Core i7-8700B, iqk vs the path that runs otherwise: ++// ne11 = 1: vec_dot; iqk was slower for iq1_m / iq2_s / iq2_xxs (0.63-0.89x) and even for the others; ++// ne11 = 2..7: iqk, 1.07-3.2x faster than vec_dot (lowest: q6_K at ne11 = 2 on Zen 4, 0.96x); ++// ne11 = 8: iqk ahead of tiled, 1.15-2.9x, except iq2_xxs / iq3_s (0.84x / 0.91x on Zen 5); ++// ne11 = 9..12: iqk for q2_K..q6_K (1.01-2.3x); the I-quants measured slower than tiled there; ++// ne11 > 12: tiled (iqk 0.16-0.98x from ne11 = 16 up; ne11 = 512: 0.19-0.91x). ++int ggml_cpu_iqk_mode(int type, int64_t ne11) { ++ if (!ggml_cpu_iqk_type_enabled(type)) { ++ return 0; ++ } ++ if (!g_iqk_auto) { ++ return g_iqk_first ? 2 : 1; ++ } ++ if (ne11 < 2) { ++ return 0; ++ } ++ if (ne11 < 8) { ++ return 2; ++ } ++ if (ne11 == 8) { ++ return type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ3_S ? 0 : 2; ++ } ++ const bool kquant = type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K || ++ type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K; ++ return ne11 <= 12 && kquant ? 2 : 0; ++} ++ +int64_t ggml_cpu_iqk_count(int type) { + return type >= 0 && type < 64 ? g_iqk_count[type].load() : 0; +} @@ -1103,10 +1133,10 @@ index 00000000..addc939b +} // extern "C" diff --git a/src/ggml-cpu/iqk/iqk_gemm.h b/src/ggml-cpu/iqk/iqk_gemm.h new file mode 100644 -index 00000000..3cf83d93 +index 0000000..a36b5d7 --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm.h -@@ -0,0 +1,41 @@ +@@ -0,0 +1,45 @@ +#pragma once + +// GEMM kernels ported from ik_llama.cpp (iqk_gemm.cpp) for the k-quant / i-quant weight types whose @@ -1121,7 +1151,7 @@ index 00000000..3cf83d93 +extern "C" { +#endif + -+// Reads once: GGML_CPU_IQK (0 disables; 1 enables, the default on ARM, while x86 defaults to off; 2 also runs them ++// Reads once: GGML_CPU_IQK (0 disables; 1 enables, the default on ARM, while x86 defaults to small batches only; 2 also runs them +// ahead of the x86 tiled K-quant path), +// GGML_CPU_IQK_TYPES (ggml type names or "all"; the other types keep the tiled / vec_dot paths) and GGML_CPU_IQK_DEBUG +// (1: print the enabled types and the first mul_mat per type to stderr). @@ -1133,6 +1163,10 @@ index 00000000..3cf83d93 +// GGML_CPU_IQK=2: the iqk kernels take the op before the tiled path gets a chance +bool ggml_cpu_iqk_before_tiled(void); + ++// For a weight type and src1 row count: 0 = not iqk, 1 = iqk after the tiled path, 2 = iqk ahead of it. ++// GGML_CPU_IQK=1 / 2 give 1 / 2 for every enabled type; unset on x86 picks by batch size (see iqk_gemm.cpp). ++int ggml_cpu_iqk_mode(int type, int64_t ne11); ++ +// C[iy*stride_C + ix] = dot(A row ix, B row iy) for ix < Nx, iy < Ny; A rows of type `type` (stride_A bytes), +// B rows are Q8_K (stride_B bytes), ne00 % 256 == 0. The work is split over the nth threads by ith. +// Returns false (computing nothing) if the type is not handled. @@ -1150,7 +1184,7 @@ index 00000000..3cf83d93 +#endif diff --git a/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp b/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp new file mode 100644 -index 00000000..43ba7450 +index 0000000..43ba745 --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm_1bit.cpp @@ -0,0 +1,364 @@ @@ -1520,7 +1554,7 @@ index 00000000..43ba7450 +#endif // IQK_IMPLEMENT diff --git a/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp b/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp new file mode 100644 -index 00000000..f916fee0 +index 0000000..f916fee --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm_iquants.cpp @@ -0,0 +1,1169 @@ @@ -2695,7 +2729,7 @@ index 00000000..f916fee0 +#endif // IQK_IMPLEMENT diff --git a/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp b/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp new file mode 100644 -index 00000000..916de2a3 +index 0000000..916de2a --- /dev/null +++ b/src/ggml-cpu/iqk/iqk_gemm_kquants.cpp @@ -0,0 +1,1162 @@ From e3c2fbd76f98821aedb9fecf32750c974d240d5e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 5 Nov 2026 07:58:33 -0800 Subject: [PATCH 5/5] ggml-patches/0018: correct the measured ranges in the iqk batch-size comment --- scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch index 4f408e4df..a9961c45f 100644 --- a/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch +++ b/scripts/unsloth/ggml-patches/0018-ggml-cpu-iqk-gemm.patch @@ -1047,8 +1047,8 @@ index 0000000..1899fd7 +// x86 default (GGML_CPU_IQK unset), from test-backend-ops MUL_MAT perf (m = 4096, k = 14336) on Zen 3 (EPYC 7763), +// Zen 4 (EPYC 9V74), Zen 5 (EPYC 9V45), Xeon 8370C / 8573C and Core i7-8700B, iqk vs the path that runs otherwise: +// ne11 = 1: vec_dot; iqk was slower for iq1_m / iq2_s / iq2_xxs (0.63-0.89x) and even for the others; -+// ne11 = 2..7: iqk, 1.07-3.2x faster than vec_dot (lowest: q6_K at ne11 = 2 on Zen 4, 0.96x); -+// ne11 = 8: iqk ahead of tiled, 1.15-2.9x, except iq2_xxs / iq3_s (0.84x / 0.91x on Zen 5); ++// ne11 = 2..7: iqk, 1.05-3.2x faster than vec_dot (lowest: q6_K at ne11 = 2 on Zen 4, 0.96x); ++// ne11 = 8: iqk ahead of tiled, 1.02-2.9x, except iq2_xxs / iq3_s (0.84x / 0.91x on Zen 5); +// ne11 = 9..12: iqk for q2_K..q6_K (1.01-2.3x); the I-quants measured slower than tiled there; +// ne11 > 12: tiled (iqk 0.16-0.98x from ne11 = 16 up; ne11 = 512: 0.19-0.91x). +int ggml_cpu_iqk_mode(int type, int64_t ne11) {