From 067221b21b96f1528145d154994463aafba0db59 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 7 Aug 2026 11:24:42 -0700 Subject: [PATCH 1/6] Fix GCC-12.x prefetch-loop collapse in greedy_search neighbor prefetch svs::lib::prefetch_l0(std::span) issues one software prefetch per cacheline to warm the next neighbor vector before the distance kernel reads it, during greedy_search graph traversal. GCC 12.x (all point releases 12.1-12.4, verified) collapses this counted loop to a SINGLE prefetch: only the first cacheline of each vector is warmed and the remaining cachelines are demand-loaded cold from DRAM. GCC 11 and GCC >=13 emit the full loop. This is a compiler codegen regression, not an SVS logic bug. Measured impact (VecSim standalone knn_query, cohere-768 fp16 IP, 1 thread, iso-recall 0.95, node = SPR Xeon 8480L, 5-rep median), gcc11 vs gcc12 vs this fix: QPS 744 / 613 / 735 (gcc12 -18%; fix recovers ~93% of the gap) L3 misses 24.7M / 856.6M / 24.3M (gcc12 ~35x; fix back to baseline) IPC 0.66 / 0.37 / 0.66 Prefetch instrs in the search worker (objdump): 7 / 3 / 5. gcc12 runs FEWER instructions yet is slower -> pure memory-latency stall from the dropped prefetch; restoring the prefetch loop removes it. Fix: force every iteration's prefetch to be emitted via volatile inline asm on x86; keep the portable _mm_prefetch path for non-x86 (#if defined(__SSE__)). Alternative for users: build with GCC != 12.x (11.x or >=13.x are unaffected). --- include/svs/lib/prefetch.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/svs/lib/prefetch.h b/include/svs/lib/prefetch.h index 0fd438ba6..79de042fb 100644 --- a/include/svs/lib/prefetch.h +++ b/include/svs/lib/prefetch.h @@ -56,9 +56,23 @@ template void prefetch_l0(std::span span) ); } + // NOTE (GCC 12.x workaround): GCC 12.x collapses this counted prefetch loop to a + // single prefetch (it drops the per-cacheline prefetches), so only the first cacheline + // of each vector is warmed and the rest are demand-loaded cold -> large L3-miss / stall + // regression in greedy_search. GCC 11 and >=13 emit the full loop. The volatile inline + // asm below forces every iteration's prefetch to be emitted on x86; the portable + // _mm_prefetch path is kept for non-x86. +#if defined(__SSE__) + for (size_t i = 0; i < num_prefetches; ++i) { + const std::byte* p = base + CACHELINE_BYTES * i; + asm volatile("prefetcht0 %0" : : "m"(*p)); + } + asm volatile("" : : "r"(num_prefetches) : "memory"); +#else for (size_t i = 0; i < num_prefetches; ++i) { prefetch_l0(base + CACHELINE_BYTES * i); } +#endif } // Default prefetching to L0 From 3405406c7bbe70c8dae81e70827179872676ffc1 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 7 Aug 2026 11:28:01 -0700 Subject: [PATCH 2/6] Use portable #pragma GCC unroll instead of x86 inline asm Replaces the x86-only inline-asm workaround with `#pragma GCC unroll` above the prefetch loop. This keeps the portable _mm_prefetch path (a no-op on non-x86), so the fix works on ALL architectures, and is a harmless hint on compilers that don't recognize the pragma. Verified: gcc12 now emits the full prefetch loop (collapsed -> restored); gcc11/13/15 unaffected; clang still emits prefetches. --- include/svs/lib/prefetch.h | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/include/svs/lib/prefetch.h b/include/svs/lib/prefetch.h index 79de042fb..7ee9f641c 100644 --- a/include/svs/lib/prefetch.h +++ b/include/svs/lib/prefetch.h @@ -56,23 +56,16 @@ template void prefetch_l0(std::span span) ); } - // NOTE (GCC 12.x workaround): GCC 12.x collapses this counted prefetch loop to a - // single prefetch (it drops the per-cacheline prefetches), so only the first cacheline - // of each vector is warmed and the rest are demand-loaded cold -> large L3-miss / stall - // regression in greedy_search. GCC 11 and >=13 emit the full loop. The volatile inline - // asm below forces every iteration's prefetch to be emitted on x86; the portable - // _mm_prefetch path is kept for non-x86. -#if defined(__SSE__) - for (size_t i = 0; i < num_prefetches; ++i) { - const std::byte* p = base + CACHELINE_BYTES * i; - asm volatile("prefetcht0 %0" : : "m"(*p)); - } - asm volatile("" : : "r"(num_prefetches) : "memory"); -#else + // GCC 12.x collapses this counted prefetch loop to a single prefetch (it drops the + // per-cacheline prefetches), leaving all but the first cacheline of each vector to be + // demand-loaded cold -> large L3-miss / memory-stall regression in greedy_search. + // GCC 11 and >=13 are unaffected. `#pragma GCC unroll` keeps the full loop under 12.x + // and is portable: it stays on the _mm_prefetch path (a no-op on non-x86) and is a + // no-op hint for compilers that don't recognize it (e.g. Clang honors it, others ignore). +#pragma GCC unroll 16 for (size_t i = 0; i < num_prefetches; ++i) { prefetch_l0(base + CACHELINE_BYTES * i); } -#endif } // Default prefetching to L0 From 47d552d33509faaca0b659c0fdfa04ca6eff6882 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 7 Aug 2026 11:40:30 -0700 Subject: [PATCH 3/6] Revert to inline-asm (the version proven to work in the built module) The #pragma GCC unroll variant fixes the isolated loop but does NOT survive the real inlining chain (accessor.prefetch -> SimpleData::prefetch -> lib::prefetch): the built module's search worker still shows the collapsed prefetch (3, same as stock gcc12). Only the volatile inline-asm version restores the prefetches in the actual binary and recovers the perf (measured). Comment updated to document this. --- include/svs/lib/prefetch.h | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/include/svs/lib/prefetch.h b/include/svs/lib/prefetch.h index 7ee9f641c..37480ca2f 100644 --- a/include/svs/lib/prefetch.h +++ b/include/svs/lib/prefetch.h @@ -56,16 +56,27 @@ template void prefetch_l0(std::span span) ); } - // GCC 12.x collapses this counted prefetch loop to a single prefetch (it drops the - // per-cacheline prefetches), leaving all but the first cacheline of each vector to be - // demand-loaded cold -> large L3-miss / memory-stall regression in greedy_search. - // GCC 11 and >=13 are unaffected. `#pragma GCC unroll` keeps the full loop under 12.x - // and is portable: it stays on the _mm_prefetch path (a no-op on non-x86) and is a - // no-op hint for compilers that don't recognize it (e.g. Clang honors it, others ignore). -#pragma GCC unroll 16 + // GCC 12.x (all point releases 12.1-12.4, verified) collapses this counted prefetch + // loop to a SINGLE prefetch: only the first cacheline of each vector is warmed and the + // rest are demand-loaded cold from DRAM -> large L3-miss / memory-stall regression in + // greedy_search. GCC 11 and >=13 are unaffected. NOTE: `#pragma GCC unroll` fixes the + // isolated loop but does NOT survive the real inlining chain (accessor.prefetch -> + // SimpleData::prefetch -> lib::prefetch) in this codebase — the collapse still happens. + // The volatile inline asm below is what actually forces every prefetch to be emitted + // in the built module (measured: restores gcc11-level cache misses and ~93% of the QPS + // gap). It is x86-only; the portable _mm_prefetch path is kept for non-x86. + // Cleanest alternative for users: build with GCC != 12.x (11.x or >=13.x). +#if defined(__SSE__) + for (size_t i = 0; i < num_prefetches; ++i) { + const std::byte* p = base + CACHELINE_BYTES * i; + asm volatile("prefetcht0 %0" : : "m"(*p)); + } + asm volatile("" : : "r"(num_prefetches) : "memory"); +#else for (size_t i = 0; i < num_prefetches; ++i) { prefetch_l0(base + CACHELINE_BYTES * i); } +#endif } // Default prefetching to L0 From 621e9024542bcebcdd383a91e1cad65126256514 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 7 Aug 2026 13:55:04 -0700 Subject: [PATCH 4/6] Use portable std::atomic_signal_fence instead of inline asm Replaces the x86 inline-asm workaround with a standard ISO C++ signal fence (std::atomic_signal_fence) inside the prefetch loop. Generates no code at runtime but stops GCC 12.x's RTL backend from folding away the per-cacheline prefetches. Why not #pragma GCC unroll: the loop survives GCC's GIMPLE passes intact in both gcc11 and gcc12 (verified via -fdump-tree); the collapse is an RTL-backend decision, so the GIMPLE-stage unroll pragma is a no-op on the hot greedy_search clone. Verified on the real VecSim/SVS module (cohere-768 fp16 IP, 1 thread, iso-recall 0.95418, 5-rep median): restores the gcc11 stride prefetch loop (worker prefetches 3->7) and QPS gcc12 721 -> 930 (gcc11 = 938), recovering ~97% of the gap. Portable across arches (no asm); Clang/ICX unaffected by the original bug. --- include/svs/lib/prefetch.h | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/include/svs/lib/prefetch.h b/include/svs/lib/prefetch.h index 37480ca2f..345817c0c 100644 --- a/include/svs/lib/prefetch.h +++ b/include/svs/lib/prefetch.h @@ -21,6 +21,7 @@ #include #include #include +#include #ifdef __SSE__ #include #endif @@ -56,27 +57,18 @@ template void prefetch_l0(std::span span) ); } - // GCC 12.x (all point releases 12.1-12.4, verified) collapses this counted prefetch - // loop to a SINGLE prefetch: only the first cacheline of each vector is warmed and the - // rest are demand-loaded cold from DRAM -> large L3-miss / memory-stall regression in - // greedy_search. GCC 11 and >=13 are unaffected. NOTE: `#pragma GCC unroll` fixes the - // isolated loop but does NOT survive the real inlining chain (accessor.prefetch -> - // SimpleData::prefetch -> lib::prefetch) in this codebase — the collapse still happens. - // The volatile inline asm below is what actually forces every prefetch to be emitted - // in the built module (measured: restores gcc11-level cache misses and ~93% of the QPS - // gap). It is x86-only; the portable _mm_prefetch path is kept for non-x86. - // Cleanest alternative for users: build with GCC != 12.x (11.x or >=13.x). -#if defined(__SSE__) - for (size_t i = 0; i < num_prefetches; ++i) { - const std::byte* p = base + CACHELINE_BYTES * i; - asm volatile("prefetcht0 %0" : : "m"(*p)); - } - asm volatile("" : : "r"(num_prefetches) : "memory"); -#else + // GCC 12.x (all point releases 12.1-12.4) drops the per-cacheline prefetches from this + // loop in the greedy_search hot path: only the first cacheline of each vector is warmed, + // the rest load cold from DRAM -> ~35x more L3 misses, ~24% slower search (GCC 11 and >=13 + // unaffected; not observed with Clang/ICX). The loop survives GCC's GIMPLE passes intact; + // the collapse is an RTL-backend decision, so #pragma GCC unroll does not help. A no-op + // signal fence per iteration is enough to stop the backend folding the prefetches away. + // Standard ISO C++ (portable, generates no code); restores the full prefetch loop on GCC 12.x + // and recovers ~97% of the lost throughput. Alternative: build with GCC != 12.x. for (size_t i = 0; i < num_prefetches; ++i) { prefetch_l0(base + CACHELINE_BYTES * i); + std::atomic_signal_fence(std::memory_order_seq_cst); } -#endif } // Default prefetching to L0 From 288960b6b9271399d779cea8e4ca45793b887b93 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 7 Aug 2026 14:20:24 -0700 Subject: [PATCH 5/6] Shorten prefetch fence comment --- include/svs/lib/prefetch.h | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/include/svs/lib/prefetch.h b/include/svs/lib/prefetch.h index 345817c0c..53b9b6638 100644 --- a/include/svs/lib/prefetch.h +++ b/include/svs/lib/prefetch.h @@ -57,14 +57,8 @@ template void prefetch_l0(std::span span) ); } - // GCC 12.x (all point releases 12.1-12.4) drops the per-cacheline prefetches from this - // loop in the greedy_search hot path: only the first cacheline of each vector is warmed, - // the rest load cold from DRAM -> ~35x more L3 misses, ~24% slower search (GCC 11 and >=13 - // unaffected; not observed with Clang/ICX). The loop survives GCC's GIMPLE passes intact; - // the collapse is an RTL-backend decision, so #pragma GCC unroll does not help. A no-op - // signal fence per iteration is enough to stop the backend folding the prefetches away. - // Standard ISO C++ (portable, generates no code); restores the full prefetch loop on GCC 12.x - // and recovers ~97% of the lost throughput. Alternative: build with GCC != 12.x. + // The fence is a no-op at runtime; without it GCC 12 drops all but the first + // prefetch here, so neighbor vectors load cold from DRAM and search slows ~24%. for (size_t i = 0; i < num_prefetches; ++i) { prefetch_l0(base + CACHELINE_BYTES * i); std::atomic_signal_fence(std::memory_order_seq_cst); From a3f6f839ac77f596d7fc6c68b77623365f52baf3 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 7 Aug 2026 17:34:27 -0700 Subject: [PATCH 6/6] clang-format: order include --- include/svs/lib/prefetch.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/svs/lib/prefetch.h b/include/svs/lib/prefetch.h index 53b9b6638..416195e93 100644 --- a/include/svs/lib/prefetch.h +++ b/include/svs/lib/prefetch.h @@ -18,10 +18,10 @@ #include "svs/lib/misc.h" +#include #include #include #include -#include #ifdef __SSE__ #include #endif