Skip to content

CPU: tinyBLAS GEMM for F16 / BF16 weights by default on x86 (VAE 1.9x, BF16 text encoder 2.4x) #47

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
danielhanchen wants to merge 5 commits into
base: cpu-sgemm-q8-0
Choose a base branch
from
Open

CPU: tinyBLAS GEMM for F16 / BF16 weights by default on x86 (VAE 1.9x, BF16 text encoder 2.4x) #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c996a5a
CPU: tinyBLAS GEMM for F16 / BF16 weights by default on x86 (GGML_CPU…
danielhanchen Nov 5, 2026
3668df7
CPU tinyBLAS: F16 / BF16 by default only on AVX512 builds and Intel A…
danielhanchen Nov 5, 2026
d1a313c
CPU tinyBLAS: keep F16 / BF16 on in MSVC AVX512 builds
danielhanchen Nov 5, 2026
f33a93d
Merge branch 'cpu-sgemm-q8-0' into cpu-sgemm-f16-bf16
danielhanchen Nov 5, 2026
f947776
CPU tinyBLAS: default F16 / BF16 on wherever measured faster or equal
danielhanchen Nov 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,9 +328,10 @@ if (NOT SD_USE_SYSTEM_GGML)
add_definitions(-DGGML_MAX_NAME=160)
endif()

# x86 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 limits it to Q8_0 weights,
# where it returns the same bits as the default vec_dot path; GGML_CPU_SGEMM_TYPES opts other types in.
if (NOT DEFINED GGML_LLAMAFILE AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64)$")
# x86 and aarch64 CPU: build llamafile's tinyBLAS GEMM (as llama.cpp does). ggml-patches/0017 picks the weight types
# it takes per CPU from measurements (Q8_0 on x86; F16 / BF16 where tinyBLAS was faster, see ggml-cpu.c);
# GGML_CPU_SGEMM_TYPES overrides the set.
if (NOT DEFINED GGML_LLAMAFILE AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64|aarch64|arm64|ARM64)$")
set(GGML_LLAMAFILE_DEFAULT ON)
endif()

Expand Down
96 changes: 86 additions & 10 deletions scripts/unsloth/ggml-patches/0017-ggml-cpu-llamafile-sgemm-q8_0-only.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,95 @@
diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c
index 30b8cbb5..160dcf72 100644
index 30b8cbb..6f9319d 100644
--- a/src/ggml-cpu/ggml-cpu.c
+++ b/src/ggml-cpu/ggml-cpu.c
@@ -49,6 +49,39 @@
@@ -49,6 +49,115 @@

#ifdef GGML_USE_LLAMAFILE
#include "llamafile/sgemm.h"
+#if defined(__AVX2__) && !defined(__AVX512F__)
+#if defined(_MSC_VER) && !defined(__clang__)
+#include <intrin.h>
+static void ggml_cpu_sgemm_cpuid(unsigned leaf, unsigned r[4]) {
+ int v[4];
+ __cpuid(v, (int) leaf);
+ for (int i = 0; i < 4; ++i) {
+ r[i] = (unsigned) v[i];
+ }
+}
+#else
+#include <cpuid.h>
+static void ggml_cpu_sgemm_cpuid(unsigned leaf, unsigned r[4]) {
+ if (!__get_cpuid(leaf, &r[0], &r[1], &r[2], &r[3])) {
+ r[0] = r[1] = r[2] = r[3] = 0;
+ }
+}
+#endif
+#endif
+
+// Weight types whose mul_mat may go through llamafile's tinyBLAS GEMM. Q8_0 by default: its tinyBLAS kernel adds
+// the per-block products in the same 8 lanes and the same final reduction as ggml_vec_dot_q8_0_q8_0, so it returns
+// the same bits, only faster. Other types change the accumulation order and are opt-in:
+// GGML_CPU_SGEMM_TYPES=q8_0,bf16,f16,f32 (ggml type names), "all" or "none".
+static uint64_t ggml_cpu_sgemm_types = 1ull << GGML_TYPE_Q8_0;
+// Weight types whose mul_mat may go through llamafile's tinyBLAS GEMM. Q8_0: its tinyBLAS kernel adds the per-block
+// products in the same 8 lanes and the same final reduction as ggml_vec_dot_q8_0_q8_0, so it returns the same bits,
+// only faster. F16 / BF16: both paths read the same F16 / BF16 activations and accumulate in F32, only in another
+// order (the text encoder, VAE and DiT F16 / BF16 matmuls). F32 doubles the rms error vs an F64 GEMM and stays
+// opt-in. GGML_CPU_SGEMM_TYPES=q8_0,bf16,f16,f32 (ggml type names), "all" or "none" overrides the default below;
+// GGML_CPU_SGEMM_TYPES=q8_0 restores the bit-identical-only set.
+//
+// Default, per measured SD1.5 F16 sampling (tinyBLAS vs vec_dot, same machine): Q8_0 on every x86 build (1.06-1.12x,
+// same bits). F16 / BF16 where tinyBLAS was faster or equal:
+// x86 AVX512 builds: Zen 5 1.11-1.16x, Xeon 8573C 1.74x (GCC) / 1.85-1.88x (MSVC);
+// x86 AVX2 builds on Intel: Xeon 8573C 1.30x (GCC) / 1.24-1.25x (MSVC), Core i7-8700B 1.45-1.6x (clang);
+// x86 AVX2 builds on AMD Zen 5 with GCC / clang (1.07-1.08x) and on Zen 4 with MSVC (1.03-1.11x);
+// aarch64 NEON builds without FP16 vector arithmetic (F32 accumulation; armv8.0 1.35x on a Neoverse N2).
+// Left off: AMD Zen 3 AVX2 (GCC 11 0.78x, MSVC 0.71x), Zen 4 AVX2 with GCC 11 (0.87-0.88x), Zen 5 AVX2 with MSVC
+// (0.86x), other AMD families (not measured), and aarch64 FP16-vector builds, whose tinyBLAS kernel accumulates in
+// FP16 rather than in another order.
+static uint64_t ggml_cpu_sgemm_types = 0;
+
+static uint64_t ggml_cpu_sgemm_types_default(void) {
+ const uint64_t f16 = (1ull << GGML_TYPE_F16) | (1ull << GGML_TYPE_BF16);
+ uint64_t mask = 0;
+#if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)
+ mask |= 1ull << GGML_TYPE_Q8_0;
+#endif
+#if defined(__AVX512F__)
+ mask |= f16;
+#elif defined(__AVX2__)
+ unsigned r[4] = { 0, 0, 0, 0 };
+ ggml_cpu_sgemm_cpuid(0, r);
+ const bool intel = r[1] == 0x756e6547 && r[3] == 0x49656e69 && r[2] == 0x6c65746e; // GenuineIntel
+ const bool amd = r[1] == 0x68747541 && r[3] == 0x69746e65 && r[2] == 0x444d4163; // AuthenticAMD
+ if (intel) {
+ mask |= f16;
+ } else if (amd) {
+ ggml_cpu_sgemm_cpuid(1, r);
+ unsigned family = (r[0] >> 8) & 0xf;
+ unsigned model = (r[0] >> 4) & 0xf;
+ if (family == 0xf) {
+ family += (r[0] >> 20) & 0xff;
+ model |= ((r[0] >> 16) & 0xf) << 4;
+ }
+ const bool zen4 = family == 0x19 && ((model >= 0x10 && model <= 0x1f) || (model >= 0x60 && model <= 0x7f) ||
+ (model >= 0xa0 && model <= 0xaf));
+ const bool zen5 = family == 0x1a;
+#if defined(_MSC_VER) && !defined(__clang__)
+ if (zen4) {
+ mask |= f16;
+ }
+ (void) zen5;
+#else
+ if (zen5) {
+ mask |= f16;
+ }
+ (void) zen4;
+#endif
+ }
+#elif defined(__ARM_NEON) && !defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+ mask |= f16;
+#endif
+ return mask;
+}
+
+static void ggml_cpu_sgemm_types_init(void) {
+ ggml_cpu_sgemm_types = ggml_cpu_sgemm_types_default();
+ const char * env = getenv("GGML_CPU_SGEMM_TYPES");
+ if (env == NULL) {
+ return;
Expand Down Expand Up @@ -42,7 +118,7 @@ index 30b8cbb5..160dcf72 100644
#endif

#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
@@ -1460,7 +1493,7 @@ void ggml_compute_forward_mul_mat(
@@ -1460,7 +1569,7 @@ void ggml_compute_forward_mul_mat(

const bool src1_cont = ggml_is_contiguous(src1);

Expand All @@ -51,7 +127,7 @@ index 30b8cbb5..160dcf72 100644
for (int64_t i13 = 0; i13 < ne13; i13++)
for (int64_t i12 = 0; i12 < ne12; i12++)
if (!llamafile_sgemm(params,
@@ -1539,7 +1572,7 @@ UseGgmlGemm1:;
@@ -1539,7 +1648,7 @@ UseGgmlGemm1:;
ggml_barrier(params->threadpool);

#if GGML_USE_LLAMAFILE
Expand All @@ -60,7 +136,7 @@ index 30b8cbb5..160dcf72 100644
const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
const size_t row_size = ggml_row_size(vec_dot_type, ne10);

@@ -4088,6 +4121,10 @@ void ggml_cpu_init(void) {
@@ -4088,6 +4197,10 @@ void ggml_cpu_init(void) {

const uint64_t t_end = ggml_time_us(); UNUSED(t_end);

Expand Down