From c996a5a096fd1d106dbe730dbbb08de4313c1c31 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 4 Nov 2026 20:29:13 -0800 Subject: [PATCH 1/4] CPU: tinyBLAS GEMM for F16 / BF16 weights by default on x86 (GGML_CPU_SGEMM_TYPES=q8_0 restores Q8_0 only) --- CMakeLists.txt | 4 ++-- ...7-ggml-cpu-llamafile-sgemm-q8_0-only.patch | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b80186560..b58ed2d04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,8 +328,8 @@ if (NOT SD_USE_SYSTEM_GGML) add_definitions(-DGGML_MAX_NAME=160) endif() -# x86 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 limits it to Q8_0 weights, -# where it returns the same bits as the default vec_dot path; GGML_CPU_SGEMM_TYPES opts other types in. +# x86 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 limits it to Q8_0 (same bits as +# the default vec_dot path), F16 and BF16 weights; GGML_CPU_SGEMM_TYPES overrides the set. if (NOT DEFINED GGML_LLAMAFILE AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64)$") set(GGML_LLAMAFILE_DEFAULT ON) endif() diff --git a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch index 577d36578..502837807 100644 --- a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch +++ b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch @@ -2,16 +2,18 @@ diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c index 30b8cbb5..160dcf72 100644 --- a/src/ggml-cpu/ggml-cpu.c +++ b/src/ggml-cpu/ggml-cpu.c -@@ -49,6 +49,39 @@ +@@ -49,6 +49,41 @@ #ifdef GGML_USE_LLAMAFILE #include "llamafile/sgemm.h" + -+// Weight types whose mul_mat may go through llamafile's tinyBLAS GEMM. Q8_0 by default: its tinyBLAS kernel adds -+// the per-block products in the same 8 lanes and the same final reduction as ggml_vec_dot_q8_0_q8_0, so it returns -+// the same bits, only faster. Other types change the accumulation order and are opt-in: -+// GGML_CPU_SGEMM_TYPES=q8_0,bf16,f16,f32 (ggml type names), "all" or "none". -+static uint64_t ggml_cpu_sgemm_types = 1ull << GGML_TYPE_Q8_0; ++// Weight types whose mul_mat may go through llamafile's tinyBLAS GEMM. Q8_0: its tinyBLAS kernel adds the per-block ++// products in the same 8 lanes and the same final reduction as ggml_vec_dot_q8_0_q8_0, so it returns the same bits, ++// only faster. F16 / BF16: both paths read the same F16 / BF16 activations and accumulate in F32, only in another ++// order (the text encoder, VAE and DiT F16 / BF16 matmuls; 1.6-2.7x on AVX2 / AVX512). F32 doubles the rms error vs ++// an F64 GEMM and stays opt-in. GGML_CPU_SGEMM_TYPES=q8_0,bf16,f16,f32 (ggml type names), "all" or "none" overrides; ++// GGML_CPU_SGEMM_TYPES=q8_0 restores the bit-identical-only set. ++static uint64_t ggml_cpu_sgemm_types = (1ull << GGML_TYPE_Q8_0) | (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); + +static void ggml_cpu_sgemm_types_init(void) { + const char * env = getenv("GGML_CPU_SGEMM_TYPES"); @@ -42,7 +44,7 @@ index 30b8cbb5..160dcf72 100644 #endif #ifdef GGML_USE_CPU_RISCV64_SPACEMIT -@@ -1460,7 +1493,7 @@ void ggml_compute_forward_mul_mat( +@@ -1460,7 +1495,7 @@ void ggml_compute_forward_mul_mat( const bool src1_cont = ggml_is_contiguous(src1); @@ -51,7 +53,7 @@ index 30b8cbb5..160dcf72 100644 for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) if (!llamafile_sgemm(params, -@@ -1539,7 +1572,7 @@ UseGgmlGemm1:; +@@ -1539,7 +1574,7 @@ UseGgmlGemm1:; ggml_barrier(params->threadpool); #if GGML_USE_LLAMAFILE @@ -60,7 +62,7 @@ index 30b8cbb5..160dcf72 100644 const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); -@@ -4088,6 +4121,10 @@ void ggml_cpu_init(void) { +@@ -4088,6 +4123,10 @@ void ggml_cpu_init(void) { const uint64_t t_end = ggml_time_us(); UNUSED(t_end); From 3668df77cfdb76c030a148ff1dbeaa51f169cdb4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 5 Nov 2026 03:50:21 -0800 Subject: [PATCH 2/4] CPU tinyBLAS: F16 / BF16 by default only on AVX512 builds and Intel AVX2 hosts On AMD the AVX2 F16 kernels ran slower than vec_dot from the release toolchains (SD1.5 F16 sampling, GitHub runners: Zen 3 0.78x and Zen 4 0.88x with GCC 11 on ubuntu-22.04, Zen 3 0.71x with MSVC), so F16 / BF16 stay opt-in there and in every MSVC build. Q8_0 stays on everywhere. GGML_CPU_SGEMM_TYPES still overrides the set. --- CMakeLists.txt | 3 +- ...7-ggml-cpu-llamafile-sgemm-q8_0-only.patch | 42 +++++++++++++++---- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b58ed2d04..52494cbb3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -329,7 +329,8 @@ if (NOT SD_USE_SYSTEM_GGML) endif() # x86 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 limits it to Q8_0 (same bits as -# the default vec_dot path), F16 and BF16 weights; GGML_CPU_SGEMM_TYPES overrides the set. +# the default vec_dot path) and, in AVX512 builds and on Intel AVX2 hosts, F16 and BF16 weights (AMD AVX2 and MSVC +# builds measured slower there); GGML_CPU_SGEMM_TYPES overrides the set. if (NOT DEFINED GGML_LLAMAFILE AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64)$") set(GGML_LLAMAFILE_DEFAULT ON) endif() diff --git a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch index 502837807..e46099845 100644 --- a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch +++ b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch @@ -1,21 +1,47 @@ diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c -index 30b8cbb5..160dcf72 100644 +index 30b8cbb..a8c7511 100644 --- a/src/ggml-cpu/ggml-cpu.c +++ b/src/ggml-cpu/ggml-cpu.c -@@ -49,6 +49,41 @@ +@@ -49,6 +49,67 @@ #ifdef GGML_USE_LLAMAFILE #include "llamafile/sgemm.h" ++#if defined(__AVX2__) && !defined(__AVX512F__) && (!defined(_MSC_VER) || defined(__clang__)) && \ ++ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64)) ++#include ++#endif + +// Weight types whose mul_mat may go through llamafile's tinyBLAS GEMM. Q8_0: its tinyBLAS kernel adds the per-block +// products in the same 8 lanes and the same final reduction as ggml_vec_dot_q8_0_q8_0, so it returns the same bits, +// only faster. F16 / BF16: both paths read the same F16 / BF16 activations and accumulate in F32, only in another -+// order (the text encoder, VAE and DiT F16 / BF16 matmuls; 1.6-2.7x on AVX2 / AVX512). F32 doubles the rms error vs -+// an F64 GEMM and stays opt-in. GGML_CPU_SGEMM_TYPES=q8_0,bf16,f16,f32 (ggml type names), "all" or "none" overrides; ++// order (the text encoder, VAE and DiT F16 / BF16 matmuls). F32 doubles the rms error vs an F64 GEMM and stays ++// opt-in. GGML_CPU_SGEMM_TYPES=q8_0,bf16,f16,f32 (ggml type names), "all" or "none" overrides the default below; +// GGML_CPU_SGEMM_TYPES=q8_0 restores the bit-identical-only set. -+static uint64_t ggml_cpu_sgemm_types = (1ull << GGML_TYPE_Q8_0) | (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); ++// ++// Default: Q8_0 everywhere (1.07-1.11x on every x86 CPU measured). F16 / BF16 only where tinyBLAS measured faster ++// than vec_dot on SD1.5 F16 sampling: AVX512 builds (Zen 5 1.11-1.16x, Xeon 8573C 1.74x) and AVX2 builds on Intel ++// (Xeon 8573C 1.30x, Core i7-8700B 1.6x). On AMD the AVX2 kernels ran slower than vec_dot from the release ++// toolchains (GCC 11: Zen 3 0.78x, Zen 4 0.88x; MSVC: Zen 3 0.71x), so F16 / BF16 stay opt-in there and in every ++// MSVC build. ++static uint64_t ggml_cpu_sgemm_types = 1ull << GGML_TYPE_Q8_0; ++ ++static uint64_t ggml_cpu_sgemm_types_default(void) { ++ uint64_t mask = 1ull << GGML_TYPE_Q8_0; ++#if !defined(_MSC_VER) || defined(__clang__) ++#if defined(__AVX512F__) ++ mask |= (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); ++#elif defined(__AVX2__) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64)) ++ unsigned int a = 0, b = 0, c = 0, d = 0; ++ if (__get_cpuid(0, &a, &b, &c, &d) && b == 0x756e6547 && d == 0x49656e69 && c == 0x6c65746e) { // GenuineIntel ++ mask |= (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); ++ } ++#endif ++#endif ++ return mask; ++} + +static void ggml_cpu_sgemm_types_init(void) { ++ ggml_cpu_sgemm_types = ggml_cpu_sgemm_types_default(); + const char * env = getenv("GGML_CPU_SGEMM_TYPES"); + if (env == NULL) { + return; @@ -44,7 +70,7 @@ index 30b8cbb5..160dcf72 100644 #endif #ifdef GGML_USE_CPU_RISCV64_SPACEMIT -@@ -1460,7 +1495,7 @@ void ggml_compute_forward_mul_mat( +@@ -1460,7 +1521,7 @@ void ggml_compute_forward_mul_mat( const bool src1_cont = ggml_is_contiguous(src1); @@ -53,7 +79,7 @@ index 30b8cbb5..160dcf72 100644 for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) if (!llamafile_sgemm(params, -@@ -1539,7 +1574,7 @@ UseGgmlGemm1:; +@@ -1539,7 +1600,7 @@ UseGgmlGemm1:; ggml_barrier(params->threadpool); #if GGML_USE_LLAMAFILE @@ -62,7 +88,7 @@ index 30b8cbb5..160dcf72 100644 const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); -@@ -4088,6 +4123,10 @@ void ggml_cpu_init(void) { +@@ -4088,6 +4149,10 @@ void ggml_cpu_init(void) { const uint64_t t_end = ggml_time_us(); UNUSED(t_end); From d1a313cf59dfb5a37b7d430bf8656e68e5e09953 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 5 Nov 2026 04:05:05 -0800 Subject: [PATCH 3/4] CPU tinyBLAS: keep F16 / BF16 on in MSVC AVX512 builds The Windows x64 bundle's AVX512 variants measured 1.88x on a Xeon 8573C (SD1.5 F16 sampling), so the MSVC exclusion now covers only the AVX2 kernels, where AMD Zen 3 / Zen 5 measured 0.71x / 0.86x. --- CMakeLists.txt | 4 ++-- ...7-ggml-cpu-llamafile-sgemm-q8_0-only.patch | 23 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52494cbb3..b50aa81e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -329,8 +329,8 @@ if (NOT SD_USE_SYSTEM_GGML) endif() # x86 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 limits it to Q8_0 (same bits as -# the default vec_dot path) and, in AVX512 builds and on Intel AVX2 hosts, F16 and BF16 weights (AMD AVX2 and MSVC -# builds measured slower there); GGML_CPU_SGEMM_TYPES overrides the set. +# the default vec_dot path) and, in AVX512 builds and GCC / clang AVX2 builds on Intel, F16 and BF16 weights (AMD +# and MSVC AVX2 builds measured slower there); GGML_CPU_SGEMM_TYPES overrides the set. if (NOT DEFINED GGML_LLAMAFILE AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64)$") set(GGML_LLAMAFILE_DEFAULT ON) endif() diff --git a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch index e46099845..5ce03a5a6 100644 --- a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch +++ b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch @@ -1,8 +1,8 @@ diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c -index 30b8cbb..a8c7511 100644 +index 30b8cbb..d4e93aa 100644 --- a/src/ggml-cpu/ggml-cpu.c +++ b/src/ggml-cpu/ggml-cpu.c -@@ -49,6 +49,67 @@ +@@ -49,6 +49,66 @@ #ifdef GGML_USE_LLAMAFILE #include "llamafile/sgemm.h" @@ -19,24 +19,23 @@ index 30b8cbb..a8c7511 100644 +// GGML_CPU_SGEMM_TYPES=q8_0 restores the bit-identical-only set. +// +// Default: Q8_0 everywhere (1.07-1.11x on every x86 CPU measured). F16 / BF16 only where tinyBLAS measured faster -+// than vec_dot on SD1.5 F16 sampling: AVX512 builds (Zen 5 1.11-1.16x, Xeon 8573C 1.74x) and AVX2 builds on Intel -+// (Xeon 8573C 1.30x, Core i7-8700B 1.6x). On AMD the AVX2 kernels ran slower than vec_dot from the release -+// toolchains (GCC 11: Zen 3 0.78x, Zen 4 0.88x; MSVC: Zen 3 0.71x), so F16 / BF16 stay opt-in there and in every -+// MSVC build. ++// than vec_dot on SD1.5 F16 sampling: AVX512 builds (Zen 5 GCC 1.11-1.16x, Xeon 8573C GCC 1.74x / MSVC 1.88x) and ++// GCC / clang AVX2 builds on Intel (Xeon 8573C 1.30x, Core i7-8700B 1.6x). On AMD the AVX2 kernels ran slower than ++// vec_dot from the release toolchains (GCC 11: Zen 3 0.78x, Zen 4 0.88x; MSVC: Zen 3 0.71x, Zen 5 0.86x), so ++// F16 / BF16 stay opt-in there and in MSVC AVX2 builds. +static uint64_t ggml_cpu_sgemm_types = 1ull << GGML_TYPE_Q8_0; + +static uint64_t ggml_cpu_sgemm_types_default(void) { + uint64_t mask = 1ull << GGML_TYPE_Q8_0; -+#if !defined(_MSC_VER) || defined(__clang__) +#if defined(__AVX512F__) + mask |= (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); -+#elif defined(__AVX2__) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64)) ++#elif defined(__AVX2__) && (!defined(_MSC_VER) || defined(__clang__)) && \ ++ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64)) + unsigned int a = 0, b = 0, c = 0, d = 0; + if (__get_cpuid(0, &a, &b, &c, &d) && b == 0x756e6547 && d == 0x49656e69 && c == 0x6c65746e) { // GenuineIntel + mask |= (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); + } +#endif -+#endif + return mask; +} + @@ -70,7 +69,7 @@ index 30b8cbb..a8c7511 100644 #endif #ifdef GGML_USE_CPU_RISCV64_SPACEMIT -@@ -1460,7 +1521,7 @@ void ggml_compute_forward_mul_mat( +@@ -1460,7 +1520,7 @@ void ggml_compute_forward_mul_mat( const bool src1_cont = ggml_is_contiguous(src1); @@ -79,7 +78,7 @@ index 30b8cbb..a8c7511 100644 for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) if (!llamafile_sgemm(params, -@@ -1539,7 +1600,7 @@ UseGgmlGemm1:; +@@ -1539,7 +1599,7 @@ UseGgmlGemm1:; ggml_barrier(params->threadpool); #if GGML_USE_LLAMAFILE @@ -88,7 +87,7 @@ index 30b8cbb..a8c7511 100644 const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); -@@ -4088,6 +4149,10 @@ void ggml_cpu_init(void) { +@@ -4088,6 +4148,10 @@ void ggml_cpu_init(void) { const uint64_t t_end = ggml_time_us(); UNUSED(t_end); From f947776929905af40f9058675b1c422307703d26 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 5 Nov 2026 07:06:30 -0800 Subject: [PATCH 4/4] CPU tinyBLAS: default F16 / BF16 on wherever measured faster or equal Adds Intel AVX2 under MSVC (1.24-1.25x), AMD Zen 4 AVX2 under MSVC (1.03-1.11x), AMD Zen 5 AVX2 under GCC / clang (1.07-1.08x) and aarch64 builds without FP16 vector arithmetic, whose tinyBLAS kernel accumulates in F32 (armv8.0 1.35x on a Neoverse N2; tinyBLAS is now built on aarch64 too). Stays off on Zen 3 AVX2 (0.71-0.78x), Zen 4 AVX2 with GCC 11 (0.87-0.88x), Zen 5 AVX2 with MSVC (0.86x) and aarch64 FP16-vector builds (FP16 accumulation). Q8_0 stays on x86 only. --- CMakeLists.txt | 8 +- ...7-ggml-cpu-llamafile-sgemm-q8_0-only.patch | 89 ++++++++++++++----- 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b50aa81e1..d8694e253 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,10 +328,10 @@ if (NOT SD_USE_SYSTEM_GGML) add_definitions(-DGGML_MAX_NAME=160) endif() -# x86 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 limits it to Q8_0 (same bits as -# the default vec_dot path) and, in AVX512 builds and GCC / clang AVX2 builds on Intel, F16 and BF16 weights (AMD -# and MSVC AVX2 builds measured slower there); GGML_CPU_SGEMM_TYPES overrides the set. -if (NOT DEFINED GGML_LLAMAFILE AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64)$") +# x86 and aarch64 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 picks the weight types +# it takes per CPU from measurements (Q8_0 on x86; F16 / BF16 where tinyBLAS was faster, see ggml-cpu.c); +# GGML_CPU_SGEMM_TYPES overrides the set. +if (NOT DEFINED GGML_LLAMAFILE AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64|aarch64|arm64|ARM64)$") set(GGML_LLAMAFILE_DEFAULT ON) endif() diff --git a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch index 5ce03a5a6..a46b48b33 100644 --- a/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch +++ b/scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch @@ -1,14 +1,29 @@ diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c -index 30b8cbb..d4e93aa 100644 +index 30b8cbb..6f9319d 100644 --- a/src/ggml-cpu/ggml-cpu.c +++ b/src/ggml-cpu/ggml-cpu.c -@@ -49,6 +49,66 @@ +@@ -49,6 +49,115 @@ #ifdef GGML_USE_LLAMAFILE #include "llamafile/sgemm.h" -+#if defined(__AVX2__) && !defined(__AVX512F__) && (!defined(_MSC_VER) || defined(__clang__)) && \ -+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64)) ++#if defined(__AVX2__) && !defined(__AVX512F__) ++#if defined(_MSC_VER) && !defined(__clang__) ++#include ++static void ggml_cpu_sgemm_cpuid(unsigned leaf, unsigned r[4]) { ++ int v[4]; ++ __cpuid(v, (int) leaf); ++ for (int i = 0; i < 4; ++i) { ++ r[i] = (unsigned) v[i]; ++ } ++} ++#else +#include ++static void ggml_cpu_sgemm_cpuid(unsigned leaf, unsigned r[4]) { ++ if (!__get_cpuid(leaf, &r[0], &r[1], &r[2], &r[3])) { ++ r[0] = r[1] = r[2] = r[3] = 0; ++ } ++} ++#endif +#endif + +// Weight types whose mul_mat may go through llamafile's tinyBLAS GEMM. Q8_0: its tinyBLAS kernel adds the per-block @@ -18,23 +33,57 @@ index 30b8cbb..d4e93aa 100644 +// opt-in. GGML_CPU_SGEMM_TYPES=q8_0,bf16,f16,f32 (ggml type names), "all" or "none" overrides the default below; +// GGML_CPU_SGEMM_TYPES=q8_0 restores the bit-identical-only set. +// -+// Default: Q8_0 everywhere (1.07-1.11x on every x86 CPU measured). F16 / BF16 only where tinyBLAS measured faster -+// than vec_dot on SD1.5 F16 sampling: AVX512 builds (Zen 5 GCC 1.11-1.16x, Xeon 8573C GCC 1.74x / MSVC 1.88x) and -+// GCC / clang AVX2 builds on Intel (Xeon 8573C 1.30x, Core i7-8700B 1.6x). On AMD the AVX2 kernels ran slower than -+// vec_dot from the release toolchains (GCC 11: Zen 3 0.78x, Zen 4 0.88x; MSVC: Zen 3 0.71x, Zen 5 0.86x), so -+// F16 / BF16 stay opt-in there and in MSVC AVX2 builds. -+static uint64_t ggml_cpu_sgemm_types = 1ull << GGML_TYPE_Q8_0; ++// Default, per measured SD1.5 F16 sampling (tinyBLAS vs vec_dot, same machine): Q8_0 on every x86 build (1.06-1.12x, ++// same bits). F16 / BF16 where tinyBLAS was faster or equal: ++// x86 AVX512 builds: Zen 5 1.11-1.16x, Xeon 8573C 1.74x (GCC) / 1.85-1.88x (MSVC); ++// x86 AVX2 builds on Intel: Xeon 8573C 1.30x (GCC) / 1.24-1.25x (MSVC), Core i7-8700B 1.45-1.6x (clang); ++// x86 AVX2 builds on AMD Zen 5 with GCC / clang (1.07-1.08x) and on Zen 4 with MSVC (1.03-1.11x); ++// aarch64 NEON builds without FP16 vector arithmetic (F32 accumulation; armv8.0 1.35x on a Neoverse N2). ++// Left off: AMD Zen 3 AVX2 (GCC 11 0.78x, MSVC 0.71x), Zen 4 AVX2 with GCC 11 (0.87-0.88x), Zen 5 AVX2 with MSVC ++// (0.86x), other AMD families (not measured), and aarch64 FP16-vector builds, whose tinyBLAS kernel accumulates in ++// FP16 rather than in another order. ++static uint64_t ggml_cpu_sgemm_types = 0; + +static uint64_t ggml_cpu_sgemm_types_default(void) { -+ uint64_t mask = 1ull << GGML_TYPE_Q8_0; ++ const uint64_t f16 = (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); ++ uint64_t mask = 0; ++#if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) ++ mask |= 1ull << GGML_TYPE_Q8_0; ++#endif +#if defined(__AVX512F__) -+ mask |= (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); -+#elif defined(__AVX2__) && (!defined(_MSC_VER) || defined(__clang__)) && \ -+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64)) -+ unsigned int a = 0, b = 0, c = 0, d = 0; -+ if (__get_cpuid(0, &a, &b, &c, &d) && b == 0x756e6547 && d == 0x49656e69 && c == 0x6c65746e) { // GenuineIntel -+ mask |= (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16); ++ mask |= f16; ++#elif defined(__AVX2__) ++ unsigned r[4] = { 0, 0, 0, 0 }; ++ ggml_cpu_sgemm_cpuid(0, r); ++ const bool intel = r[1] == 0x756e6547 && r[3] == 0x49656e69 && r[2] == 0x6c65746e; // GenuineIntel ++ const bool amd = r[1] == 0x68747541 && r[3] == 0x69746e65 && r[2] == 0x444d4163; // AuthenticAMD ++ if (intel) { ++ mask |= f16; ++ } else if (amd) { ++ ggml_cpu_sgemm_cpuid(1, r); ++ unsigned family = (r[0] >> 8) & 0xf; ++ unsigned model = (r[0] >> 4) & 0xf; ++ if (family == 0xf) { ++ family += (r[0] >> 20) & 0xff; ++ model |= ((r[0] >> 16) & 0xf) << 4; ++ } ++ const bool zen4 = family == 0x19 && ((model >= 0x10 && model <= 0x1f) || (model >= 0x60 && model <= 0x7f) || ++ (model >= 0xa0 && model <= 0xaf)); ++ const bool zen5 = family == 0x1a; ++#if defined(_MSC_VER) && !defined(__clang__) ++ if (zen4) { ++ mask |= f16; ++ } ++ (void) zen5; ++#else ++ if (zen5) { ++ mask |= f16; ++ } ++ (void) zen4; ++#endif + } ++#elif defined(__ARM_NEON) && !defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER) ++ mask |= f16; +#endif + return mask; +} @@ -69,7 +118,7 @@ index 30b8cbb..d4e93aa 100644 #endif #ifdef GGML_USE_CPU_RISCV64_SPACEMIT -@@ -1460,7 +1520,7 @@ void ggml_compute_forward_mul_mat( +@@ -1460,7 +1569,7 @@ void ggml_compute_forward_mul_mat( const bool src1_cont = ggml_is_contiguous(src1); @@ -78,7 +127,7 @@ index 30b8cbb..d4e93aa 100644 for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) if (!llamafile_sgemm(params, -@@ -1539,7 +1599,7 @@ UseGgmlGemm1:; +@@ -1539,7 +1648,7 @@ UseGgmlGemm1:; ggml_barrier(params->threadpool); #if GGML_USE_LLAMAFILE @@ -87,7 +136,7 @@ index 30b8cbb..d4e93aa 100644 const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); -@@ -4088,6 +4148,10 @@ void ggml_cpu_init(void) { +@@ -4088,6 +4197,10 @@ void ggml_cpu_init(void) { const uint64_t t_end = ggml_time_us(); UNUSED(t_end);