From 0a865ab06916accbc1de8171e7a072cd857794e4 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Tue, 27 Oct 2026 08:38:27 -0400 Subject: [PATCH] fix: load FP8 weights as F16 on Metal instead of aborting ggml-metal reports MUL_MAT with an FP8 src0 as supported but has no FP8 matrix kernel, so Linear keeps FP8 weights and the process aborts in new_sd_ctx (ggml-metal-device.cpp: not implemented). A carried patch to ModelManager::register_param_tensors retypes FP8 weights that a Metal backend computes with to F16 before they are allocated; the loader's existing conversion fills them. F16 holds every E4M3 and E5M2 value. Other backends keep FP8. tests/test_fp8_runtime.py generates from an FP8 copy of the 12 MB PixArt test model on each device and compares it with an F32 control; it fails on Metal without the patch. CI runs it on the Linux CPU and lavapipe Vulkan targets and the macOS CPU. Fixes #29 --- .github/workflows/validate.yml | 8 + CMakeLists.txt | 1 + README.md | 5 +- docs/upstream_1017_qualification.md | 5 + patches/README.md | 1 + patches/fp8_metal_f16_load.patch | 56 +++++++ tests/native/fp8_runtime_test.cpp | 91 ++++++++++++ tests/test_fp8_runtime.py | 221 ++++++++++++++++++++++++++++ 8 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 patches/fp8_metal_f16_load.patch create mode 100644 tests/native/fp8_runtime_test.cpp create mode 100644 tests/test_fp8_runtime.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index df1516f..80c609b 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -128,6 +128,14 @@ jobs: env: SD_RUNTIME_API_TARGET: ${{ matrix.target }} SD_REQUIRE_RUNTIME: "1" + - name: Generate from FP8 weights + # Metal needs a Mac: GitHub's macOS runners list no Metal device. + if: runner.os != 'Windows' && matrix.smoke + run: python3 -m unittest discover -s tests -p test_fp8_runtime.py -v + env: + SD_FP8_TARGET: ${{ matrix.target }} + SD_REQUIRE_RUNTIME: "1" + GGML_VK_VISIBLE_DEVICES: "0" - uses: dart-lang/setup-dart@v1 if: runner.os == 'Linux' && matrix.smoke - name: Verify cooperative host shutdown diff --git a/CMakeLists.txt b/CMakeLists.txt index 90bb552..2ea0751 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,7 @@ function(sd_native_carry_patch patch) endfunction() sd_native_carry_patch(pickle_line_span.patch src/model_io/pickle_io.cpp) +sd_native_carry_patch(fp8_metal_f16_load.patch src/model_manager.cpp) # Wrapper exports are compiled into the upstream library target, so they share # its SD_API export macro and reach upstream without patching it. diff --git a/README.md b/README.md index 31a365f..71ccb4e 100644 --- a/README.md +++ b/README.md @@ -610,9 +610,10 @@ the macOS probe and builds the iOS device and simulator slices. Upstream 1017 has its [ABI audit, wrapper review, FP8 evidence and outstanding gates](docs/upstream_1017_qualification.md); the carry inventory is in the [upstream 982 record](docs/upstream_982_qualification.md). -It carries two fixes the pin lacks: unbatched CFG in Metal builds (below) and +It carries three fixes the pin lacks: unbatched CFG in Metal builds (below), the [`pickle_line_span.patch`](patches/README.md) for legacy checkpoints above -2 GiB. +2 GiB, and the [`fp8_metal_f16_load.patch`](patches/README.md), which loads FP8 +weights as F16 on Metal instead of aborting. Upstream's `generate_image` now returns an owned result with per-image generation metadata; `sd_dart_generate_image` keeps returning an image array that `free_sd_images` frees and drops that metadata. Since upstream 982 a null diff --git a/docs/upstream_1017_qualification.md b/docs/upstream_1017_qualification.md index 83ae61d..87d7850 100644 --- a/docs/upstream_1017_qualification.md +++ b/docs/upstream_1017_qualification.md @@ -113,6 +113,11 @@ This is a pre-existing defect of every published Apple runtime, not caused or fixed by this upgrade, and the default Apple backend crashes rather than refusing. Vulkan was not run. +`patches/fp8_metal_f16_load.patch` ([#29](https://github.com/leehack/stable-diffusion-native/issues/29)) +fixes it after this upgrade: under a debugger the abort is in +`ggml_metal_op_mul_mat` during the SD2 v-prediction probe in `new_sd_ctx`, and +Metal now loads FP8 weights as F16. + ## Executed qualification macOS arm64 (Apple M4 Max), release build of this branch, dylib SHA-256 diff --git a/patches/README.md b/patches/README.md index d91826f..c24be11 100644 --- a/patches/README.md +++ b/patches/README.md @@ -13,3 +13,4 @@ an upstream issue or pull request to retire it. | Patch | Sources | Fixes | Test | | --- | --- | --- | --- | | `pickle_line_span.patch` | `src/model_io/pickle_io.cpp` | A legacy PyTorch checkpoint above 2 GiB is rejected: the pickle scanner narrows the rest of the file to `int` before looking for a newline ([#23](https://github.com/leehack/stable-diffusion-native/issues/23)). | `tests/test_pickle_span.py` | +| `fp8_metal_f16_load.patch` | `src/model_manager.cpp` | An FP8 (E4M3 or E5M2) checkpoint aborts the process on Metal: ggml-metal claims FP8 `MUL_MAT` but has no FP8 kernel. Weights a Metal backend computes with now load as F16, which holds every FP8 value; other backends keep FP8 ([#29](https://github.com/leehack/stable-diffusion-native/issues/29)). | `tests/test_fp8_runtime.py` | diff --git a/patches/fp8_metal_f16_load.patch b/patches/fp8_metal_f16_load.patch new file mode 100644 index 0000000..20a03b5 --- /dev/null +++ b/patches/fp8_metal_f16_load.patch @@ -0,0 +1,56 @@ +diff --git a/src/model_manager.cpp b/src/model_manager.cpp +--- a/src/model_manager.cpp ++++ b/src/model_manager.cpp +@@ -2,6 +2,7 @@ + + #include + #include ++#include + #include + #include + #include +@@ -185,6 +186,36 @@ size_t estimate_tensors_size(const std::map& tensors) + return size; + } + ++// ggml-metal reports FP8 MUL_MAT as supported, so Linear keeps FP8 weights, but ++// it has no FP8 matrix kernel and aborts the process encoding one. Weights a ++// Metal backend computes with are loaded as F16 instead, which holds every FP8 ++// value exactly. ++static void load_fp8_params_as_f16_on_metal(const std::map& tensors, ++ ggml_backend_t compute_backend) { ++#ifndef SD_USE_UPSTREAM_GGML ++ ggml_backend_dev_t device = compute_backend != nullptr ? ggml_backend_get_device(compute_backend) : nullptr; ++ ggml_backend_reg_t reg = device != nullptr ? ggml_backend_dev_backend_reg(device) : nullptr; ++ if (reg == nullptr || std::strcmp(ggml_backend_reg_name(reg), "MTL") != 0) { ++ return; ++ } ++ for (const auto& pair : tensors) { ++ ggml_tensor* tensor = pair.second; ++ if (tensor == nullptr || (tensor->type != GGML_TYPE_F8_E4M3 && tensor->type != GGML_TYPE_F8_E5M2) || ++ tensor->data != nullptr || tensor->buffer != nullptr || tensor->view_src != nullptr) { ++ continue; ++ } ++ tensor->type = GGML_TYPE_F16; ++ tensor->nb[0] = ggml_type_size(GGML_TYPE_F16); ++ for (int i = 1; i < GGML_MAX_DIMS; ++i) { ++ tensor->nb[i] = tensor->nb[i - 1] * tensor->ne[i - 1]; ++ } ++ } ++#else ++ (void)tensors; ++ (void)compute_backend; ++#endif ++} ++ + void ModelManager::set_split_buffer_type(ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft, const std::vector>& device_limits) { + if (compute_backend == nullptr) { + return; +@@ -228,6 +259,7 @@ bool ModelManager::register_param_tensors(ModelComponent component, + LOG_ERROR("model manager tensor component is invalid"); + return false; + } ++ load_fp8_params_as_f16_on_metal(tensors, compute_backend); + if (registered_tensor_size != nullptr) { + *registered_tensor_size += estimate_tensors_size(tensors); + } diff --git a/tests/native/fp8_runtime_test.cpp b/tests/native/fp8_runtime_test.cpp new file mode 100644 index 0000000..09912ee --- /dev/null +++ b/tests/native/fp8_runtime_test.cpp @@ -0,0 +1,91 @@ +// Loads a model and generates one image through the guarded wrapper exports, +// as llamadart does, and writes its RGB bytes. +// +// Usage: fp8_runtime_test devices +// fp8_runtime_test generate +// +// The model is one that `exit_teardown_runtime_test make-model` wrote, or an +// FP8 copy of it: it holds its TAESD decoder, so it needs no VAE. + +#include "sd_dart_wrapper.h" + +#include +#include +#include +#include + +namespace { + +const int kImageSize = 64; + +int list_devices() { + std::vector devices(sd_list_devices(nullptr, 0) + 1); + sd_list_devices(devices.data(), devices.size()); + std::printf("%s", devices.data()); + return 0; +} + +int generate(const char* model, const char* backend, const char* output) { + sd_dart_log_enable(); + sd_ctx_params_t context_params; + sd_ctx_params_init(&context_params); + context_params.model_path = model; + context_params.taesd_path = model; + context_params.backend = backend; + sd_ctx_t* context = sd_dart_new_sd_ctx(&context_params); + if (context == nullptr) { + std::vector error(4096); + sd_dart_last_error(error.data(), error.size()); + std::fprintf(stderr, "load failed: %s\n", error.data()); + return 3; + } + + sd_img_gen_params_t params; + sd_img_gen_params_init(¶ms); + params.prompt = "a lighthouse"; + params.negative_prompt = ""; + params.width = kImageSize; + params.height = kImageSize; + params.seed = 7; + params.sample_params.sample_steps = 2; + params.sample_params.guidance.txt_cfg = 1.0f; + sd_image_t* images = nullptr; + int count = 0; + if (!sd_dart_generate_image(context, ¶ms, &images, &count)) { + std::vector error(4096); + sd_dart_last_error(error.data(), error.size()); + std::fprintf(stderr, "generation failed: %s\n", error.data()); + sd_dart_exit_free(context); + return 4; + } + int status = 0; + if (images == nullptr || count != 1 || images[0].width != kImageSize || + images[0].height != kImageSize || images[0].channel != 3) { + std::fprintf(stderr, "unexpected images\n"); + status = 5; + } else { + const size_t bytes = static_cast(kImageSize) * kImageSize * 3; + FILE* file = std::fopen(output, "wb"); + if (file == nullptr || std::fwrite(images[0].data, 1, bytes, file) != bytes || + std::fclose(file) != 0) { + std::fprintf(stderr, "cannot write %s\n", output); + status = 6; + } + } + free_sd_images(images, count); + sd_dart_exit_free(context); + return status; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc == 2 && std::strcmp(argv[1], "devices") == 0) { + return list_devices(); + } + if (argc == 5 && std::strcmp(argv[1], "generate") == 0) { + return generate(argv[2], argv[3], argv[4]); + } + std::fprintf(stderr, "usage: %s devices | generate \n", argv[0]); + return 2; +} diff --git a/tests/test_fp8_runtime.py b/tests/test_fp8_runtime.py new file mode 100644 index 0000000..b2f5709 --- /dev/null +++ b/tests/test_fp8_runtime.py @@ -0,0 +1,221 @@ +"""Generates from FP8 E4M3 weights on every device of a built runtime. + +The model is the 12 MB PixArt model `exit_teardown_runtime_test make-model` +writes, with its weights replaced by fixed patterns. Its 2-D weights hold +values E4M3 represents exactly, kept as F32 in a control copy and stored as +F8_E4M3 in a second copy, so on each device the two images differ only by +matrix precision. Nothing is downloaded. + +On Metal `patches/fp8_metal_f16_load.patch` loads FP8 weights as F16; without +it the FP8 generation aborts the process (#29). GitHub's macOS runners list +no Metal device; run this on a Mac with `SD_REQUIRE_METAL_GENERATION=1`. + + SD_FP8_TARGET= a built Linux target; macOS defaults to + the host's + SD_REQUIRE_RUNTIME=1 fail instead of skipping + SD_REQUIRE_METAL_GENERATION=1 fail when no Metal device generates the + control +""" + +from __future__ import annotations + +import json +import os +import platform +import shutil +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "tools")) + +import build # noqa: E402 + +COMPILER = os.environ.get("CXX") or shutil.which("c++") +MODEL_WRITER = REPO_ROOT / "tests" / "native" / "exit_teardown_runtime_test.cpp" +PROBE = REPO_ROOT / "tests" / "native" / "fp8_runtime_test.cpp" +GGML_METAL = REPO_ROOT / "third_party" / "stable-diffusion.cpp" / "ggml" / "src" / "ggml-metal" +# Mean absolute difference per channel byte between the FP8 and F32 images of +# one device. An M4 Max measured 0.03 on the CPU and 0 on Metal. +MAX_MEAN_DIFFERENCE = 0.5 +# Fewer distinct bytes would mean a flat image that hides any difference. +MIN_DISTINCT_BYTES = 16 +# 16 E4M3 values, +-2^-6 * (1 + m/8): sign, exponent field 1, mantissa m. +E4M3_PATTERN = bytes(sign | 0x08 | m for sign in (0, 0x80) for m in range(8)) +# Every other tensor: values in [-0.125, 0.125). Larger ones make the tiny +# network chaotic, so that matrix precision alone changes the image. +OTHER_PATTERN = struct.pack("<61f", *(((i * 37) % 61 - 30.5) / 61 * 0.25 for i in range(61))) + + +def e4m3_value(byte: int) -> float: + value = 2.0 ** -6 * (1 + (byte & 0x07) / 8) + return -value if byte & 0x80 else value + + +def read_safetensors(path: Path) -> tuple[dict, bytes]: + data = path.read_bytes() + (size,) = struct.unpack(" None: + offset = 0 + for name, info in header.items(): + info["data_offsets"] = [offset, offset + len(blobs[name])] + offset += len(blobs[name]) + text = json.dumps(header, separators=(",", ":")).encode() + text += b" " * (-len(text) % 8) + with path.open("wb") as file: + file.write(struct.pack(" int: + """Writes the F32 control and FP8 copies of `source`; returns the FP8 count.""" + header, _ = read_safetensors(source) + header.pop("__metadata__", None) + control_header, fp8_header, control_blobs, fp8_blobs = {}, {}, {}, {} + pattern_f32 = struct.pack(f"<{len(E4M3_PATTERN)}f", *map(e4m3_value, E4M3_PATTERN)) + converted = 0 + for name, info in header.items(): + if info["dtype"] != "F32": + raise ValueError(f"{name} is {info['dtype']}, not F32") + begin, end = info["data_offsets"] + count = (end - begin) // 4 + control_header[name] = dict(info) + fp8_header[name] = dict(info) + if len(info["shape"]) == 2 and name.endswith(".weight"): + repeats = -(-count // len(E4M3_PATTERN)) + control_blobs[name] = (pattern_f32 * repeats)[:4 * count] + fp8_blobs[name] = (E4M3_PATTERN * repeats)[:count] + fp8_header[name]["dtype"] = "F8_E4M3" + converted += 1 + else: + repeats = -(-count // 61) + control_blobs[name] = fp8_blobs[name] = (OTHER_PATTERN * repeats)[:4 * count] + write_safetensors(control, control_header, control_blobs) + write_safetensors(fp8, fp8_header, fp8_blobs) + return converted + + +def mean_difference(first: bytes, second: bytes) -> float: + return sum(abs(a - b) for a, b in zip(first, second)) / len(first) + + +class Fp8RuntimeTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.skip_reason = "" + target_name = os.environ.get("SD_FP8_TARGET", "") + if not target_name and sys.platform == "darwin": + target_name = "macos-arm64" if platform.machine() == "arm64" else "macos-x64" + if not target_name or sys.platform == "win32": + cls.skip_reason = "set SD_FP8_TARGET to a built Linux target" + return + target = build.TARGETS[target_name] + library = build.BIN_ROOT / target.name / "lib" / target.library + if not library.is_file() or COMPILER is None: + cls.skip_reason = f"no built runtime for {target.name} or no C++ compiler" + return + cls.directory = tempfile.TemporaryDirectory() + work = Path(cls.directory.name) + binaries = {} + for source in (MODEL_WRITER, PROBE): + binary = work / source.stem + subprocess.run( + [COMPILER, "-std=c++17", "-O1", "-pthread", f"-I{REPO_ROOT / 'src'}", + f"-I{build.HEADER.parent}", str(source), f"-L{library.parent}", + "-lstable-diffusion", f"-Wl,-rpath,{library.parent}", "-o", str(binary)], + check=True) + binaries[source] = binary + cls.probe = binaries[PROBE] + source_model = work / "model.safetensors" + subprocess.run([str(binaries[MODEL_WRITER]), "make-model", str(source_model)], + check=True, timeout=600, capture_output=True) + cls.control = work / "control.safetensors" + cls.fp8 = work / "fp8.safetensors" + cls.converted = write_fp8_pair(source_model, cls.control, cls.fp8) + listing = subprocess.run([str(cls.probe), "devices"], check=True, capture_output=True, + text=True, timeout=120).stdout + cls.devices = [line.split("\t")[0] for line in listing.splitlines() if line.strip()] + + @classmethod + def tearDownClass(cls) -> None: + if hasattr(cls, "directory"): + cls.directory.cleanup() + + def setUp(self) -> None: + if self.skip_reason: + if os.environ.get("SD_REQUIRE_RUNTIME"): + self.fail(self.skip_reason) + self.skipTest(self.skip_reason) + + def generate(self, model: Path, device: str) -> tuple[subprocess.CompletedProcess, bytes]: + output = Path(self.directory.name) / f"{model.stem}-{device}.rgb" + output.unlink(missing_ok=True) + result = subprocess.run([str(self.probe), "generate", str(model), device, str(output)], + capture_output=True, text=True, timeout=600) + return result, output.read_bytes() if output.is_file() else b"" + + def check_device(self, device: str) -> None: + control, control_image = self.generate(self.control, device) + if control.returncode != 0: + if device.startswith("MTL") and os.environ.get("SD_REQUIRE_METAL_GENERATION"): + self.fail(f"the F32 control does not generate on {device}:\n{control.stderr}") + self.skipTest(f"{device} cannot generate the F32 control:\n{control.stderr[-2000:]}") + fp8, fp8_image = self.generate(self.fp8, device) + self.assertEqual(0, fp8.returncode, f"FP8 weights on {device}:\n{fp8.stderr[-4000:]}") + self.assertGreaterEqual(len(set(control_image)), MIN_DISTINCT_BYTES) + self.assertEqual(len(control_image), len(fp8_image)) + difference = mean_difference(control_image, fp8_image) + self.assertLessEqual(difference, MAX_MEAN_DIFFERENCE, + f"the FP8 image on {device} is not the F32 control's") + + def test_model_carries_fp8_weights(self) -> None: + self.assertGreater(self.converted, 0) + header, _ = read_safetensors(self.fp8) + self.assertIn("F8_E4M3", {info.get("dtype") for info in header.values()}) + + def test_fp8_generates_on_the_cpu(self) -> None: + self.check_device("CPU") + + def test_fp8_generates_on_each_gpu(self) -> None: + gpus = [device for device in self.devices if device != "CPU"] + if sys.platform == "darwin" and not any(d.startswith("MTL") for d in gpus) and \ + os.environ.get("SD_REQUIRE_METAL_GENERATION"): + self.fail(f"no Metal device in {self.devices}") + if not gpus: + self.skipTest(f"no GPU device in {self.devices}") + for device in gpus: + with self.subTest(device=device): + self.check_device(device) + + +class PinnedGgmlMetalTest(unittest.TestCase): + # The patch can go once ggml-metal multiplies FP8 matrices or stops + # claiming that it does. Checked on the sources, so it runs without a GPU. + def test_pinned_ggml_metal_still_needs_the_patch(self) -> None: + metal = GGML_METAL + if not (metal / "ggml-metal-device.m").is_file(): + if os.environ.get("CI"): + self.fail("third_party/stable-diffusion.cpp is not checked out") + self.skipTest("third_party/stable-diffusion.cpp is not checked out") + supports = (metal / "ggml-metal-device.m").read_text() + start = supports.index("static bool ggml_metal_supports_mul_mat_op(") + claim = supports[start:supports.index("\n}\n", start)] + kernels = "".join(path.read_text() for path in (metal / "kernels").glob("*.metal")) + self.assertNotIn("F8_E", claim, "ggml-metal now excludes FP8 from MUL_MAT; " + "re-evaluate patches/fp8_metal_f16_load.patch") + self.assertNotRegex(kernels, r"kernel_mul_m[mv]\w*_f8", + "ggml-metal now has an FP8 matrix kernel; " + "re-evaluate patches/fp8_metal_f16_load.patch") + + +if __name__ == "__main__": + unittest.main()