From 4d6acac7721ea9bc9513ddb200a5b0cafd20771a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 17:51:02 -0700 Subject: [PATCH 1/3] Vulkan: decide direct convolution auto from the output check, not a timing race --- docs/backend.md | 8 ++++---- docs/backend_probe.md | 5 +++-- src/core/probe/conv_direct.cpp | 21 +++++++++++++++++---- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index 057d3f20ef..b1775470b3 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -233,10 +233,10 @@ sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on - `on` requests direct 2D and 3D convolution. - `off` uses the existing im2col-based convolution paths. - `auto` first checks support for a representative F16-weight/F32-input direct - convolution graph. Vulkan then benchmarks direct convolution against im2col - using two small shapes per dimension, with warmup, output validation, and five - interleaved measurements per shape. Direct convolution must improve the sum of - median times by at least 5%; a failed probe defaults to `off`. + convolution graph. On Vulkan, direct 2D convolution is then enabled when its + output on two small shapes matches im2col (a failed probe defaults to `off`), + and direct 3D convolution stays `off`. The decision does not depend on timing, + so the same seed renders the same pixels on a busy and on an idle machine. Other backends retain their implicit GEMM checks without benchmarking: CUDA 2D requires a compiled Turing-or-newer MMA architecture; HIP 2D requires the supported CDNA MFMA or RDNA WMMA architectures described below for flash diff --git a/docs/backend_probe.md b/docs/backend_probe.md index c420326d4c..e51a9b96ad 100644 --- a/docs/backend_probe.md +++ b/docs/backend_probe.md @@ -5,8 +5,9 @@ policy stays separate: `src/core/probe/flash_attn.cpp`, for example, defines att shapes and its CPU/CUDA/HIP/Vulkan shortcuts, then calls `sd_run_backend_probe` for unknown configurations. `src/core/probe/conv_direct.cpp` uses metadata-only probe graphs to check direct 2D/3D convolution support, then enables known implicit GEMM -paths on non-Vulkan backends. Vulkan compares direct convolution with im2col -using the shared measurement machinery. +paths on non-Vulkan backends. Vulkan runs direct 2D convolution against im2col +through the shared measurement machinery for its output check only (the timing +is logged, not used), and keeps direct 3D convolution off. For each representative shape, create a `BackendProbeCase`: diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp index 7f4666a41f..60091b802c 100644 --- a/src/core/probe/conv_direct.cpp +++ b/src/core/probe/conv_direct.cpp @@ -65,6 +65,15 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { return false; } if (sd_backend_is(backend, "Vulkan")) { + // The decision must not depend on how busy the machine is. The two probe shapes run in well under a + // millisecond, so on a loaded host both paths measure the same submit overhead and the old 5% race + // flipped from run to run, changing the pixels of a fixed seed. Direct 3D convolution stays off: it is + // a tie on NVIDIA in the probe and fails its output check on RADV. Direct 2D convolution is on whenever its + // output matches im2col: measured faster and far smaller in VAE decodes on RADV (gfx1151) and NVIDIA. + if (conv3d) { + LOG_INFO("conv3d direct auto: %s -> off (Vulkan)", name); + return false; + } std::vector cases; for (const auto& shape : {std::pair{32, 64}, {16, 128}}) { BackendProbeCase test; @@ -74,14 +83,18 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { test.initialize = [conv3d](BackendProbeGraph& graph) { return initialize_conv_probe(graph, conv3d); }; cases.push_back(std::move(test)); } - const auto result = sd_run_backend_probe(backend, 0, cases); + // Only the output check decides, so one timed run each and no time budget a busy host could exhaust. + BackendProbeOptions options; + options.measurement_runs = 1; + options.time_budget_ms = 1e9; + const auto result = sd_run_backend_probe(backend, 0, cases, options); if (!result.completed) { LOG_WARN("conv%dd direct auto: %s -> off (%s)", dims, name, result.error.c_str()); return false; } - LOG_INFO("conv%dd direct auto: %s -> %s (probe: im2col %.3f ms, direct %.3f ms)", - dims, name, result.candidate_faster ? "on" : "off", result.reference_ms, result.candidate_ms); - return result.candidate_faster; + LOG_INFO("conv%dd direct auto: %s -> on (output check passed; probe: im2col %.3f ms, direct %.3f ms)", + dims, name, result.reference_ms, result.candidate_ms); + return true; } if (sd_backend_supports_conv_implicit_gemm(backend, conv3d)) { LOG_INFO("conv%dd direct auto: %s -> on (implicit GEMM)", dims, name); From a2c3b975a261d745d7a5ff9f5020c602c9a2df70 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 22:10:54 -0700 Subject: [PATCH 2/3] Vulkan conv direct auto: no free-memory floor on the output-check probe The probe graphs need a few MB; the default 128 MiB floor still turned direct convolution off whenever another workload held the VRAM, so load could change the path and the pixels. A device too full to allocate the probe still fails it (off). Also update docs/performance.md, which still described the Vulkan rule as a benchmark. --- docs/performance.md | 5 +++-- src/core/probe/conv_direct.cpp | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/performance.md b/docs/performance.md index 18491039eb..eb23ca5423 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -32,8 +32,9 @@ segment count change. path. Performance depends on GPU architecture, backend, driver, model, and tensor shapes, so enabling direct convolution can either improve or reduce speed. -Automatic selection uses small benchmarks on Vulkan and implicit GEMM checks -on other backends. It does not guarantee the fastest end-to-end generation. +Automatic selection checks direct 2D convolution output against im2col on +Vulkan (timing is not used; direct 3D stays off there) and uses implicit GEMM +checks on other backends. It does not guarantee the fastest end-to-end generation. Compare `on` and `off` with the same model and generation settings over repeated runs, keeping flash attention unchanged and separating startup/probe time from steady-state generation time. See [backend selection](backend.md) diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp index 60091b802c..5e1688553a 100644 --- a/src/core/probe/conv_direct.cpp +++ b/src/core/probe/conv_direct.cpp @@ -84,9 +84,13 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { cases.push_back(std::move(test)); } // Only the output check decides, so one timed run each and no time budget a busy host could exhaust. + // No free-memory cutoff either: the probe graphs need a few MB, and the default 128 MiB floor would turn + // direct convolution off whenever another workload holds the VRAM. A device too full to allocate them + // still fails the probe (allocation failure -> off). BackendProbeOptions options; - options.measurement_runs = 1; - options.time_budget_ms = 1e9; + options.measurement_runs = 1; + options.time_budget_ms = 1e9; + options.min_free_device_bytes = 0; const auto result = sd_run_backend_probe(backend, 0, cases, options); if (!result.completed) { LOG_WARN("conv%dd direct auto: %s -> off (%s)", dims, name, result.error.c_str()); From 3d7baa21881be53bf91abb5f304a74d6164f7a20 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 5 Nov 2026 04:37:35 -0800 Subject: [PATCH 3/3] Vulkan conv direct auto: VAE only Direct 2D convolution decodes faster in a fraction of the memory, but in UNet sampling (SD1.5, 512 px) it ran 11% slower than im2col on AMD's Windows driver (Radeon 8060S) and tied on RADV. Auto now turns it on for the VAE when the output check passes; the diffusion model and ControlNet keep im2col unless asked (--conv-direct auto,diffusion=on). --- docs/backend.md | 10 ++++++---- docs/performance.md | 5 +++-- src/pipeline/diffusion_engine.cpp | 12 ++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index b1775470b3..a3baf8731a 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -233,10 +233,12 @@ sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on - `on` requests direct 2D and 3D convolution. - `off` uses the existing im2col-based convolution paths. - `auto` first checks support for a representative F16-weight/F32-input direct - convolution graph. On Vulkan, direct 2D convolution is then enabled when its - output on two small shapes matches im2col (a failed probe defaults to `off`), - and direct 3D convolution stays `off`. The decision does not depend on timing, - so the same seed renders the same pixels on a busy and on an idle machine. + convolution graph. On Vulkan, direct 2D convolution is then enabled for the + VAE when its output on two small shapes matches im2col (a failed probe + defaults to `off`); the diffusion model and ControlNet keep im2col (direct ran + slower in UNet sampling on AMD's Windows driver), and direct 3D convolution + stays `off`. The decision does not depend on timing, so the same seed renders + the same pixels on a busy and on an idle machine. Other backends retain their implicit GEMM checks without benchmarking: CUDA 2D requires a compiled Turing-or-newer MMA architecture; HIP 2D requires the supported CDNA MFMA or RDNA WMMA architectures described below for flash diff --git a/docs/performance.md b/docs/performance.md index eb23ca5423..11a639215c 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -33,8 +33,9 @@ path. Performance depends on GPU architecture, backend, driver, model, and tensor shapes, so enabling direct convolution can either improve or reduce speed. Automatic selection checks direct 2D convolution output against im2col on -Vulkan (timing is not used; direct 3D stays off there) and uses implicit GEMM -checks on other backends. It does not guarantee the fastest end-to-end generation. +Vulkan and uses it for the VAE (timing is not used; the diffusion model keeps +im2col and direct 3D stays off there) and uses implicit GEMM checks on other +backends. It does not guarantee the fastest end-to-end generation. Compare `on` and `off` with the same model and generation settings over repeated runs, keeping flash attention unchanged and separating startup/probe time from steady-state generation time. See [backend selection](backend.md) diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 75a234542f..62f59bc164 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -1272,6 +1272,18 @@ bool StableDiffusionGGML::conv_direct_for(SDBackendModule module, bool conv3d) { if (any_enabled && !enabled) { LOG_INFO("conv%dd direct auto: %s -> off (compute devices disagree)", conv3d ? 3 : 2, sd_backend_module_name(module)); } + // Vulkan: auto turns direct 2D convolution on for the VAE only. There it decodes faster in a fraction of the + // memory (1.2x on AMD's Windows driver, 1.5x on RADV, 6.7x at 1024 px), but in UNet sampling (SD1.5, 512 px) it + // ran 11% slower than im2col on AMD's Windows driver and tied on RADV, so the diffusion model and ControlNet keep + // im2col unless asked (--conv-direct auto,diffusion=on). + if (enabled && !conv3d && module != SDBackendModule::VAE) { + for (auto backend : backends) { + if (sd_backend_is(backend, "Vulkan")) { + LOG_INFO("conv2d direct auto: %s -> off (Vulkan: VAE only)", sd_backend_module_name(module)); + return false; + } + } + } return enabled; }