From 7c5b3addac59a576efeb0c74d3823fb81b7a5d27 Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 4 Nov 2026 22:40:55 +0800 Subject: [PATCH 1/2] feat: add P-H3-Image generation and editing support --- README.md | 1 + docs/p_h3_image.md | 34 ++++++++ examples/common/common.cpp | 8 +- examples/server/recipes/defaults.md | 1 + examples/server/recipes/p-h3-image.json | 55 +++++++++++++ src/conditioning/conditioner.hpp | 64 ++++++++++++--- src/core/layer_split_partition.cpp | 2 +- src/core/vulkan/shaders/modulate.comp | 19 +++-- src/core/vulkan/shaders/modulate_convrot.comp | 4 +- src/model.h | 3 +- src/model/integrations/minimax_h3.cpp | 9 +++ src/model/te/ph3_image_connector.hpp | 70 ++++++++++++++++ src/model/vae/minimax_h3_vae.hpp | 79 ++++++++++++++++--- src/model_loader.cpp | 19 +++++ src/name_conversion.cpp | 7 ++ src/pipeline/diffusion_engine.cpp | 18 +++-- src/pipeline/image.cpp | 57 ++++++++++++- src/pipeline/model_builders.cpp | 7 ++ 18 files changed, 417 insertions(+), 40 deletions(-) create mode 100644 docs/p_h3_image.md create mode 100644 examples/server/recipes/p-h3-image.json create mode 100644 src/model/te/ph3_image_connector.hpp diff --git a/README.md b/README.md index 1337c5b92f..28af00a697 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ API and command-line option may change frequently.*** - [Z-Image](./docs/z_image.md) - [Z-Image L2P](./docs/z_image_l2p.md) - [MiniT2I](./docs/minit2i.md) + - [P-H3-Image](./docs/p_h3_image.md) - [SenseNova U1.5](./docs/sensenova_u1.md) - [Ovis-Image](./docs/ovis_image.md) - [Anima](./docs/anima.md) diff --git a/docs/p_h3_image.md b/docs/p_h3_image.md new file mode 100644 index 0000000000..20de6fb202 --- /dev/null +++ b/docs/p_h3_image.md @@ -0,0 +1,34 @@ +# How to Use + +P-H3-Image supports text-to-image and reference-image editing. It uses a MiniMax-H3 diffusion transformer, a single-frame VAE, and Qwen3-VL-4B with a text connector. Prompt enhancement is not included. + +## Download weights + +- Download P-H3-Image + - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/diffusion_models +- Download P-H3-Image VAE + - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/vae +- Download Qwen3-VL-4B with the text connector + - https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/text_encoders + +Alternatively, reuse Qwen3-VL-4B, including its vision tower, with the [separate text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors). Save the connector as `p-h3-image_text_connector.safetensors` and pass it with `--embeddings-connectors`. + +Use the P-H3 image VAE; the MiniMax-H3 video VAE is not compatible. No audio VAE is needed. + +## Examples + +### Text-to-image + +``` +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -p "a lovely cat holding a sign says 'p-h3-image.cpp'" -v --offload-to-cpu --rng cpu -H 2912 -W 2912 +``` + +### Image editing + +``` +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -r source.png -r reference.png -p "Edit . Give the cat the scarf from , preserving its face and the background." -v --offload-to-cpu --rng cpu -H 2912 -W 2912 +``` + +Reference images are numbered in argument order as ``, ``, and so on. A plain edit instruction gets `Edit . ` prepended automatically. Prompts starting with `Edit ` or `subject_definitions:` are used directly. + +Use dimensions divisible by 32. The upstream default is 2912 × 2912; smaller canvases reduce memory use. Use CFG 1 and `--sigmas p-h3-image` for the upstream 8-step schedule. Keep the default VAE tiling size of 256 pixels. diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 637bbb659f..d5496f66fe 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -464,7 +464,7 @@ ArgOptions SDContextParams::get_options() { &uncond_diffusion_model_path}, {"", "--embeddings-connectors", - "path to LTXAV embeddings connectors", + "path to embeddings connectors (LTXAV or P-H3-Image)", 0, &embeddings_connectors_path}, {"", @@ -1530,6 +1530,8 @@ ArgOptions SDGenerationParams::get_options() { std::string sigmas_str = value; if (sigmas_str == "qwen-image-2.1-turbo") { sigmas_str = "1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0"; + } else if (sigmas_str == "p-h3-image") { + sigmas_str = "1.0,0.984753,0.967905,0.951688,0.925372,0.881923,0.810083,0.528718,0.0"; } if (!sigmas_str.empty() && sigmas_str.front() == '[') { sigmas_str.erase(0, 1); @@ -1777,11 +1779,11 @@ ArgOptions SDGenerationParams::get_options() { on_scheduler_arg}, {"", "--sigmas", - "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo.", + "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo, p-h3-image.", on_sigmas_arg}, {"", "--hires-sigmas", - "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo.", + "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo, p-h3-image.", on_hires_sigmas_arg}, {"", "--skip-layers", diff --git a/examples/server/recipes/defaults.md b/examples/server/recipes/defaults.md index e20a2cd783..efe596c92b 100644 --- a/examples/server/recipes/defaults.md +++ b/examples/server/recipes/defaults.md @@ -71,6 +71,7 @@ For community edit adapters and other optional extensions, these are the base-mo | `minit2i` | 2.5 | 20 | 512 × 512 | [Source](https://huggingface.co/MiniT2I/MiniT2I/raw/main/pipeline.py) B/16 example and native model size; L/16 uses CFG 6. | | `moss-image-base` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base) | | `moss-image-sft` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT) | +| `p-h3-image` | 1 | 8 | 1024 × 1024 | [Runtime guide](../../../docs/p_h3_image.md) Fixed P-H3 sigma schedule; upstream 2912 × 2912 reduced to the catalog default. | | `nitrosd-realism` | 1 | 1 | 1024 × 1024 | [Source](https://huggingface.co/ChenDY/NitroFusion) | | `ovis-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/ATH-MaaS/Ovis-Image-7B) | | `pid` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | diff --git a/examples/server/recipes/p-h3-image.json b/examples/server/recipes/p-h3-image.json new file mode 100644 index 0000000000..1098bab9eb --- /dev/null +++ b/examples/server/recipes/p-h3-image.json @@ -0,0 +1,55 @@ +{ + "schema_version": 1, + "id": "p-h3-image", + "name": "P-H3-Image", + "size_multiple": 32, + "description": "Supports text-to-image and reference-image editing with the fixed 8-step schedule. Use the P-H3 single-frame VAE and Qwen3-VL-4B with its vision tower. Supply the P-H3 text connector separately unless it is included in the text encoder.", + "model_url": "https://huggingface.co/PrunaAI/P-H3-Image", + "slots": [ + { + "id": "diffusion_model", + "required": true + }, + { + "id": "llm", + "required": true, + "label": "Qwen 3 VL 4B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision projector (required when not included in the text encoder)" + }, + { + "id": "embeddings_connectors", + "required": false, + "label": "P-H3 Text Connector (required when not included in the text encoder)" + }, + { + "id": "vae", + "required": true, + "label": "P-H3 Image VAE" + } + ], + "defaults": { + "sampling_method": "euler", + "cfg_scale": 1, + "steps": 8, + "width": 1024, + "height": 1024, + "custom_sigmas": [ + 1.0, + 0.984753, + 0.967905, + 0.951688, + 0.925372, + 0.881923, + 0.810083, + 0.528718, + 0.0 + ] + }, + "features": [ + "ref_images" + ] +} diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index e3832a1d8e..7c8b7492f9 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -17,6 +17,7 @@ #include "model/te/llada_image_te.hpp" #include "model/te/llm.hpp" #include "model/te/ming_image_te.hpp" +#include "model/te/ph3_image_connector.hpp" #include "model/te/t5.hpp" #include "model_loader.h" #include "tokenizers/sensenova_u1_tokenizer.h" @@ -1976,6 +1977,7 @@ struct LLMEmbedder : public Conditioner { std::shared_ptr tokenizer; std::shared_ptr llm; std::shared_ptr byt5; + std::shared_ptr ph3_connector; LLMEmbedder(ggml_backend_t backend, const String2TensorStorage& tensor_storage_map = {}, @@ -2022,12 +2024,18 @@ struct LLMEmbedder : public Conditioner { } else if (version == VERSION_MOSS_IMAGE) { arch = LLM::LLMArch::QWEN3_5; } - llm = std::make_shared(arch, - backend, - tensor_storage_map, - "text_encoders.llm", - enable_vision, - weight_manager); + llm = std::make_shared(arch, + backend, + tensor_storage_map, + "text_encoders.llm", + enable_vision, + weight_manager); + if (version == VERSION_P_H3_IMAGE) { + if (llm->config.num_layers != 36 || llm->config.hidden_size != 2560) { + throw std::runtime_error("P-H3-Image requires the Qwen3-VL-4B text encoder"); + } + ph3_connector = std::make_shared(backend, tensor_storage_map, weight_manager); + } int pad_id = 151643; if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) { pad_id = 11; @@ -2063,6 +2071,9 @@ struct LLMEmbedder : public Conditioner { void get_param_tensors(std::map& tensors) override { llm->get_param_tensors(tensors, "text_encoders.llm"); + if (ph3_connector) { + ph3_connector->get_param_tensors(tensors); + } if (byt5) { byt5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer"); } @@ -2074,6 +2085,9 @@ struct LLMEmbedder : public Conditioner { void set_max_graph_vram_bytes(size_t max_vram_bytes) override { llm->set_max_graph_vram_bytes(max_vram_bytes); + if (ph3_connector) { + ph3_connector->set_max_graph_vram_bytes(max_vram_bytes); + } if (byt5) { byt5->set_max_graph_vram_bytes(max_vram_bytes); } @@ -2081,6 +2095,9 @@ struct LLMEmbedder : public Conditioner { void set_runtime_backends(const std::vector& backends) override { llm->set_runtime_backends(backends); + if (ph3_connector) { + ph3_connector->set_runtime_backends(backends); + } if (byt5) { byt5->set_runtime_backends(backends); } @@ -2113,6 +2130,9 @@ struct LLMEmbedder : public Conditioner { void set_flash_attention_enabled(bool enabled) override { llm->set_flash_attention_enabled(enabled); + if (ph3_connector) { + ph3_connector->set_flash_attention_enabled(enabled); + } if (byt5) { byt5->set_flash_attention_enabled(enabled); } @@ -2120,12 +2140,18 @@ struct LLMEmbedder : public Conditioner { void set_scale_overrides(float linear_scale, float attn_scale) override { llm->set_scale_overrides(linear_scale, attn_scale); + if (ph3_connector) { + ph3_connector->set_scale_overrides(linear_scale, attn_scale); + } if (byt5) { byt5->set_scale_overrides(linear_scale, attn_scale); } } void set_weight_adapter(const std::shared_ptr& adapter) override { + if (ph3_connector) { + ph3_connector->set_weight_adapter(adapter); + } if (llm) { llm->set_weight_adapter(adapter); } @@ -2135,6 +2161,9 @@ struct LLMEmbedder : public Conditioner { } void runner_end() override { + if (ph3_connector) { + ph3_connector->runner_end(); + } if (llm) { llm->runner_end(); } @@ -2311,6 +2340,12 @@ struct LLMEmbedder : public Conditioner { LOG_ERROR("LLM prompt encoding failed"); return {}; } + if (ph3_connector) { + hidden_states = ph3_connector->compute(n_threads, hidden_states); + if (hidden_states.empty()) { + return {}; + } + } hidden_states = apply_token_weights(std::move(hidden_states), weights); GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx); @@ -2457,8 +2492,9 @@ struct LLMEmbedder : public Conditioner { if (sd_version_is_minimax_h3(version)) { prompt_template_encode_start_idx = 0; - out_layers = {50}; - prompt_attn_range = {0, 0}; + // The final tap is normalized; LLMRunner exposes it at num_layers + 1. + out_layers = version == VERSION_P_H3_IMAGE ? std::set{13, 19, 25, 31, 37} : std::set{50}; + prompt_attn_range = {0, 0}; if (llm->enable_vision) { const int patch_size = llm->config.vision.patch_size; @@ -2564,7 +2600,17 @@ struct LLMEmbedder : public Conditioner { } } } - prompt += conditioner_params.text; + auto text = conditioner_params.text; + if (version == VERSION_P_H3_IMAGE && !image_embeds.empty()) { + // Preserve explicit picture targets and structured briefs from the upstream upsampler. + static const std::regex verbatim(R"(^\s*(Edit |subject_definitions:))"); + if (!std::regex_search(text, verbatim)) { + text = "Edit . " + text; + } else { + text = trim(text); + } + } + prompt += text; } else if (sd_version_is_hunyuan_video(version)) { prompt_template_encode_start_idx = 98; out_layers = {26}; diff --git a/src/core/layer_split_partition.cpp b/src/core/layer_split_partition.cpp index 6ef24357a4..ff8aaeb602 100644 --- a/src/core/layer_split_partition.cpp +++ b/src/core/layer_split_partition.cpp @@ -206,7 +206,7 @@ namespace sd { for (size_t seg_idx = 0; seg_idx < plan.segments.size(); seg_idx++) { ggml_backend_t backend = backend_by_segment[seg_idx]; const auto& segment = plan.segments[seg_idx]; - auto assign_node = [&](int node_index) { + auto assign_node = [&](int node_index) { if (node_index < 0 || node_index >= n_nodes) { return; } diff --git a/src/core/vulkan/shaders/modulate.comp b/src/core/vulkan/shaders/modulate.comp index df07626d2a..81f910d07b 100644 --- a/src/core/vulkan/shaders/modulate.comp +++ b/src/core/vulkan/shaders/modulate.comp @@ -1,16 +1,25 @@ #version 450 layout(local_size_x = 256) in; -layout(binding = 0) readonly buffer X { float x[]; }; -layout(binding = 1) readonly buffer S { float scale[]; }; -layout(binding = 2) readonly buffer H { float shift[]; }; -layout(binding = 3) writeonly buffer D { float dst[]; }; +layout(binding = 0) readonly buffer X { + float x[]; +}; +layout(binding = 1) readonly buffer S { + float scale[]; +}; +layout(binding = 2) readonly buffer H { + float shift[]; +}; +layout(binding = 3) writeonly buffer D { + float dst[]; +}; layout(push_constant) uniform Parameters { uint ne[4], scale_ne[4], shift_ne[4]; uint x_stride[4], scale_stride[4], shift_stride[4]; uint offsets[4]; uint count, has_shift; -} p; +} +p; void main() { uint group = gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x; diff --git a/src/core/vulkan/shaders/modulate_convrot.comp b/src/core/vulkan/shaders/modulate_convrot.comp index 3cac110c93..4e85b1d0b8 100644 --- a/src/core/vulkan/shaders/modulate_convrot.comp +++ b/src/core/vulkan/shaders/modulate_convrot.comp @@ -2,8 +2,8 @@ #extension GL_EXT_control_flow_attributes : require -layout(constant_id = 0) const uint K = 4096; -layout(constant_id = 1) const bool AFFINE = false; +layout(constant_id = 0) const uint K = 4096; +layout(constant_id = 1) const bool AFFINE = false; layout(constant_id = 2) const bool HAS_SHIFT = true; layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; layout(binding = 0) readonly buffer X { diff --git a/src/model.h b/src/model.h index 12bc22e0f0..4695f5c74d 100644 --- a/src/model.h +++ b/src/model.h @@ -70,6 +70,7 @@ enum SDVersion { VERSION_MOSS_IMAGE, VERSION_FIBO2, VERSION_CHROMA, + VERSION_P_H3_IMAGE, VERSION_COUNT, }; @@ -166,7 +167,7 @@ static inline bool sd_version_is_ltxav(SDVersion version) { } static inline bool sd_version_is_minimax_h3(SDVersion version) { - return version == VERSION_MINIMAX_H3; + return version == VERSION_MINIMAX_H3 || version == VERSION_P_H3_IMAGE; } static inline bool sd_version_is_wan(SDVersion version) { diff --git a/src/model/integrations/minimax_h3.cpp b/src/model/integrations/minimax_h3.cpp index 2569b626ea..35a1f6700b 100644 --- a/src/model/integrations/minimax_h3.cpp +++ b/src/model/integrations/minimax_h3.cpp @@ -40,6 +40,15 @@ const std::vector& minimax_h3_definitions() { definition.build = build_minimax_h3; result.push_back(std::move(definition)); } + { + ModelDefinition definition = result.front(); + definition.version = VERSION_P_H3_IMAGE; + definition.name = "P-H3-Image"; + definition.traits.supports_video_generation = false; + definition.sampling.default_flow_shift = 1.f; + definition.ref_image_preset = "qwen"; + result.push_back(std::move(definition)); + } return result; }(); return definitions; diff --git a/src/model/te/ph3_image_connector.hpp b/src/model/te/ph3_image_connector.hpp new file mode 100644 index 0000000000..1e74e8e765 --- /dev/null +++ b/src/model/te/ph3_image_connector.hpp @@ -0,0 +1,70 @@ +#ifndef __SD_MODEL_TE_PH3_IMAGE_CONNECTOR_HPP__ +#define __SD_MODEL_TE_PH3_IMAGE_CONNECTOR_HPP__ + +#include "core/ggml_runner.h" +#include "model/common/ggml_block.hpp" + +struct PH3ImageTextConnector : public UnaryBlock { + static constexpr int64_t input_dim = 5 * 2560; + static constexpr int64_t output_dim = 5120; + + PH3ImageTextConnector() { + blocks["proj"] = std::make_shared(input_dim, output_dim, false, true, true); + } + + void init_params(ggml_context* ctx, + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "") override { + for (const auto* name : {"input_mean", "input_std"}) { + params[name] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, input_dim); + } + for (const auto* name : {"output_mean", "output_std", "sink_embedding"}) { + params[name] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, output_dim); + } + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { + auto proj = std::dynamic_pointer_cast(blocks["proj"]); + x = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, x, params["input_mean"]), params["input_std"]); + x = proj->forward(ctx, x); + x = ggml_add(ctx->ggml_ctx, ggml_mul(ctx->ggml_ctx, x, params["output_std"]), params["output_mean"]); + auto sink = ggml_repeat(ctx->ggml_ctx, params["sink_embedding"], ggml_ext_slice(ctx->ggml_ctx, x, 1, 0, 1)); + if (x->ne[1] == 1) { + return sink; + } + return ggml_concat(ctx->ggml_ctx, sink, ggml_ext_slice(ctx->ggml_ctx, x, 1, 1, x->ne[1]), 1); + } +}; + +struct PH3ImageTextConnectorRunner : public GGMLRunner { + PH3ImageTextConnector model; + static constexpr const char* prefix = "text_encoders.ph3_image_connector"; + + PH3ImageTextConnectorRunner(ggml_backend_t backend, + const String2TensorStorage& tensor_storage_map, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager) { + model.init(params_ctx, tensor_storage_map, prefix); + } + + std::string get_desc() override { + return "ph3_image_text_connector"; + } + + void get_param_tensors(std::map& tensors) { + model.get_param_tensors(tensors, prefix); + } + + sd::Tensor compute(int n_threads, const sd::Tensor& hidden_states) { + auto get_graph = [&]() { + auto gf = ggml_new_graph(compute_ctx); + auto x = make_input(hidden_states); + auto runner_ctx = get_context(); + ggml_build_forward_expand(gf, model.forward(&runner_ctx, x)); + return gf; + }; + return take_or_empty(GGMLRunner::compute(get_graph, n_threads, false)); + } +}; + +#endif // __SD_MODEL_TE_PH3_IMAGE_CONNECTOR_HPP__ diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp index 1e06558a19..d5bb01a4db 100644 --- a/src/model/vae/minimax_h3_vae.hpp +++ b/src/model/vae/minimax_h3_vae.hpp @@ -10,6 +10,7 @@ #include #include +#include "core/rng_mt19937.hpp" #include "model/common/rope.hpp" #include "model/diffusion/dit.hpp" #include "model/vae/vae.hpp" @@ -18,6 +19,34 @@ namespace MiniMaxH3VAE { constexpr int H3_VIDEO_VAE_GRAPH_SIZE = 262144; + struct MiniMaxH3VAEConfig { + int num_layers = 36; + int patch_size_t = 4; + + static MiniMaxH3VAEConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) { + MiniMaxH3VAEConfig config; + auto proj = tensors.find(prefix + ".decoder.proj_out.weight"); + if (proj != tensors.end()) { + config.patch_size_t = static_cast(proj->second.ne[1] / (3 * 16 * 16)); + if (config.patch_size_t != 1 && config.patch_size_t != 4) { + throw std::runtime_error("unsupported MiniMax-H3 VAE temporal patch size"); + } + int layers = 0; + const auto layer_prefix = prefix + ".decoder.transformer_blocks."; + for (const auto& [name, _] : tensors) { + if (starts_with(name, layer_prefix)) { + layers = std::max(layers, std::atoi(name.c_str() + layer_prefix.size()) + 1); + } + } + if (layers > 0) { + config.num_layers = layers; + } + LOG_VERBOSE("minimax_h3_vae: layers=%d, patch_size_t=%d", config.num_layers, config.patch_size_t); + } + return config; + } + }; + struct CausalConv3d : public Conv3d { std::tuple temporal_padding; @@ -361,13 +390,14 @@ namespace MiniMaxH3VAE { }; struct Decoder : public GGMLBlock { - static constexpr int dim = 2048; - static constexpr int num_layers = 36; + static constexpr int dim = 2048; + int num_layers; static constexpr int num_register_tokens = 4; static constexpr int patch_size = 16; - static constexpr int patch_size_t = 4; + int patch_size_t; - Decoder() { + explicit Decoder(const MiniMaxH3VAEConfig& config) + : num_layers(config.num_layers), patch_size_t(config.patch_size_t) { blocks["x_embedder"] = std::make_shared(24, dim, true); for (int i = 0; i < num_layers; ++i) { blocks["transformer_blocks." + std::to_string(i)] = @@ -389,7 +419,9 @@ namespace MiniMaxH3VAE { GGML_TYPE_F32, dim, num_register_tokens); - params["mask_token"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim); + if (patch_size_t != 1) { + params["mask_token"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim); + } } ggml_tensor* forward(GGMLRunnerContext* ctx, @@ -443,7 +475,10 @@ namespace MiniMaxH3VAE { }; struct MiniMaxH3VideoVAE : public GGMLBlock { - MiniMaxH3VideoVAE() { + bool single_frame; + + explicit MiniMaxH3VideoVAE(const MiniMaxH3VAEConfig& config) + : single_frame(config.patch_size_t == 1) { blocks["encoder"] = std::make_shared(); blocks["quant_conv"] = std::make_shared(48, 48, @@ -451,7 +486,7 @@ namespace MiniMaxH3VAE { blocks["post_quant_conv"] = std::make_shared(24, 24, std::tuple{1, 1, 1}); - blocks["decoder"] = std::make_shared(); + blocks["decoder"] = std::make_shared(config); } ggml_tensor* encode(GGMLRunnerContext* ctx, @@ -464,7 +499,7 @@ namespace MiniMaxH3VAE { auto encoder = std::dynamic_pointer_cast(blocks["encoder"]); auto quant = std::dynamic_pointer_cast(blocks["quant_conv"]); auto moments = quant->forward(ctx, encoder->forward(ctx, pixels)); - return ggml_ext_slice(ctx->ggml_ctx, moments, 3, 0, 24); + return single_frame ? moments : ggml_ext_slice(ctx->ggml_ctx, moments, 3, 0, 24); } ggml_tensor* decode(GGMLRunnerContext* ctx, @@ -483,6 +518,7 @@ namespace MiniMaxH3VAE { }; struct MiniMaxH3VideoVAERunner : public VAE { + MiniMaxH3VAEConfig config; MiniMaxH3VideoVAE model; sd::Tensor pixel_mean; sd::Tensor pixel_std; @@ -495,6 +531,8 @@ namespace MiniMaxH3VAE { const std::string& prefix = "first_stage_model", std::shared_ptr weight_manager = nullptr) : VAE(VERSION_MINIMAX_H3, backend, prefix, weight_manager), + config(MiniMaxH3VAEConfig::detect_from_weights(tensor_storage_map, prefix)), + model(config), pixel_mean({1, 1, 1, 3}, {0.485f, 0.456f, 0.406f}), pixel_std({1, 1, 1, 3}, {0.229f, 0.224f, 0.225f}), latents_mean({1, 1, 1, 24}, @@ -516,12 +554,12 @@ namespace MiniMaxH3VAE { } std::string get_desc() override { - return "minimax_h3_video_vae"; + return config.patch_size_t == 1 ? "p_h3_image_vae" : "minimax_h3_video_vae"; } int get_encoder_output_channels(int input_channels) override { SD_UNUSED(input_channels); - return 24; + return config.patch_size_t == 1 ? 48 : 24; } void get_param_tensors(std::map& tensors) override { @@ -531,6 +569,15 @@ namespace MiniMaxH3VAE { sd::Tensor vae_output_to_latents(const sd::Tensor& vae_output, std::shared_ptr rng) override { SD_UNUSED(rng); + if (config.patch_size_t == 1) { + auto mean = sd::ops::slice(vae_output, 3, 0, 24); + auto logvar = sd::ops::slice(vae_output, 3, 24, 48); + auto noise = MT19937RNG(42).randn(static_cast(mean.numel())); + for (int64_t i = 0; i < mean.numel(); ++i) { + mean[i] += std::exp(0.5f * std::clamp(logvar[i], -30.f, 20.f)) * noise[i]; + } + return mean; + } return vae_output; } @@ -610,6 +657,10 @@ namespace MiniMaxH3VAE { bool circular_y = false) override { auto input = ensure_video_shape(x); auto tiling = resolve_tiling_params(tiling_params); + if (config.patch_size_t == 1 && input.shape()[2] != 1) { + LOG_ERROR("P-H3-Image VAE encodes single-frame images only"); + return {}; + } if (input.shape()[2] == 1) { auto encoded = VAE::encode(n_threads, input, tiling, circular_x, circular_y); if (!encoded.empty() && encoded.shape()[2] > 1) { @@ -654,6 +705,10 @@ namespace MiniMaxH3VAE { bool silent = false) override { auto input = ensure_video_shape(x); auto tiling = resolve_tiling_params(tiling_params); + if (config.patch_size_t == 1 && input.shape()[2] != 1) { + LOG_ERROR("P-H3-Image VAE decodes single-frame latents only"); + return {}; + } if (input.shape()[2] == 1) { auto decoded = VAE::decode(n_threads, input, @@ -752,7 +807,9 @@ namespace MiniMaxH3VAE { ids.reserve(static_cast(width * height * num_frames + 5)); constexpr float two_pi = 6.28318530717958647692f; for (int64_t t = 0; t < num_frames; ++t) { - float pt = (2.f * ((t + 0.5f) / num_frames) - 1.f) * two_pi; + float pt = (config.patch_size_t == 1 ? 2.f * (0.5f / 7.f) - 1.f + : 2.f * ((t + 0.5f) / num_frames) - 1.f) * + two_pi; for (int64_t h = 0; h < height; ++h) { float ph = (2.f * ((h + 0.5f) / height) - 1.f) * two_pi; for (int64_t w = 0; w < width; ++w) { diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 5576067409..407885bbff 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -578,6 +578,25 @@ SDVersion ModelLoader::get_sd_version() const { } if (tensor_storage.name.find("model.diffusion_model.video_patch_proj.weight") != std::string::npos && tensor_storage_map.find("model.diffusion_model.audio_patch_proj.weight") != tensor_storage_map.end()) { + // P-H3 uses the same DiT architecture as H3; its VAE or connector identifies the image pipeline. + auto decoder = tensor_storage_map.find("first_stage_model.decoder.proj_out.weight"); + if (decoder == tensor_storage_map.end()) { + decoder = tensor_storage_map.find("vae.decoder.proj_out.weight"); + } + if (decoder != tensor_storage_map.end() && decoder->second.ne[1] == 768) { + return VERSION_P_H3_IMAGE; + } + for (const std::string prefix : {"", "embeddings_connectors."}) { + if (tensor_storage_map.count(prefix + "text_encoders.ph3_image_connector.proj.weight") || + tensor_storage_map.count(prefix + "text_encoders.llm.text_connector.proj.weight")) { + return VERSION_P_H3_IMAGE; + } + } + auto connector = tensor_storage_map.find("embeddings_connectors.proj.weight"); + if (connector != tensor_storage_map.end() && connector->second.ne[0] == 12800 && + connector->second.ne[1] == 5120 && tensor_storage_map.count("embeddings_connectors.sink_embedding")) { + return VERSION_P_H3_IMAGE; + } return VERSION_MINIMAX_H3; } if (name == "model.diffusion_model.y_embedder.layer_pool.weight" && diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index 57a41c5c39..2e147348ff 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -1699,12 +1699,19 @@ std::string convert_tensor_name(std::string name, SDVersion version) { } } + const std::vector> connector_prefix_map = { + {"embeddings_connectors.text_encoders.", "text_encoders."}, + {"embeddings_connectors.", version == VERSION_P_H3_IMAGE ? "text_encoders.ph3_image_connector." : ""}, + }; + replace_with_prefix_map(name, connector_prefix_map); + std::unordered_map prefix_map = { {"diffusion_model.", "model.diffusion_model."}, {"unet.", "model.diffusion_model."}, {"transformer.", "model.diffusion_model."}, // dit {"vae.", "first_stage_model."}, {"text_encoders.llm.text_embedding_projection.", "text_embedding_projection."}, + {"text_encoders.llm.text_connector.", "text_encoders.ph3_image_connector."}, {"text_encoder.", "cond_stage_model.transformer."}, {"te.", "cond_stage_model.transformer."}, {"text_encoder.2.", "cond_stage_model.1.transformer."}, diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index d161d9e5be..98e80e1174 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -778,7 +778,7 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf if (strlen(SAFE_STR(sd_ctx_params->embeddings_connectors_path)) > 0) { LOG_INFO("loading embeddings connectors from '%s'", sd_ctx_params->embeddings_connectors_path); - if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path)) { + if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path, "embeddings_connectors.")) { LOG_WARN("loading embeddings connectors from '%s' failed", sd_ctx_params->embeddings_connectors_path); } } @@ -1491,7 +1491,7 @@ bool StableDiffusionGGML::build_denoiser() { } case H3_AV_FLOW_PRED: { LOG_INFO("running in MiniMax H3 AV FLOW mode"); - denoiser = std::make_shared(default_flow_shift, 3.f, get_latent_channel()); + denoiser = std::make_shared(default_flow_shift, version == VERSION_P_H3_IMAGE ? 1.f : 3.f, get_latent_channel()); break; } case MING_IMAGE_FLOW_PRED: { @@ -2594,9 +2594,9 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr StableDiffusionGGML::decode_first_stage(const sd::Tensordiffusion_to_vae_latents(x); + auto image_latents = version == VERSION_P_H3_IMAGE && x.dim() == 5 + ? sd::ops::slice(x, 3, 0, get_latent_channel()) + : x; + auto latents = first_stage_model->diffusion_to_vae_latents(image_latents); auto tiling_params = first_stage_model->resolve_tiling_params(vae_tiling_params); const bool prefer_temporal_tiling = decode_video && latents.dim() == 5 && latents.shape()[2] > 1 && first_stage_model->can_temporal_tile_decode(); @@ -3021,6 +3024,9 @@ sd::Tensor StableDiffusionGGML::decode_first_stage(const sd::Tensordecode(n_threads, latents, tiling_params, decode_video, circular_x, circular_y); + if (version == VERSION_P_H3_IMAGE && !decoded.empty() && decoded.dim() == 5) { + decoded = decoded.squeeze(2); + } if (!decoded.empty() || !sd::backend_fit::prepare_vae_decode_retry_tiling(tiling_params, prefer_temporal_tiling, first_stage_model->last_compute_status(), @@ -3220,7 +3226,7 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima if (params.force_ref_timestep_zero && !sd_version_is_krea2(version)) { LOG_WARN("force_ref_timestep_zero is only supported by Krea2 architecture for now"); } - if (sd_version_is_minimax_h3(version)) { + if (version == VERSION_MINIMAX_H3) { // H3 consumes references as VAE latents in its own packed sequence rather than // through the generic conditioner/DiT reference path, so ref_image_size is the // only arg it acts on. Everything else is reported as ineffective rather than diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 607ec18c5e..0763e38ca0 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -144,6 +144,34 @@ namespace sd::pipeline { return padded; } + static sd::Tensor resize_ph3_reference(const sd::Tensor& image, int64_t pixels) { + double scale = std::min(1.0, std::sqrt(static_cast(pixels) / (image.shape()[0] * image.shape()[1]))); + int64_t width = std::max(32, static_cast(std::nearbyint(image.shape()[0] * scale / 32)) * 32); + int64_t height = std::max(32, static_cast(std::nearbyint(image.shape()[1] * scale / 32)) * 32); + if (width == image.shape()[0] && height == image.shape()[1]) { + return image; + } + return sd::ops::interpolate(image, {width, height, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Lanczos); + } + + static sd::Tensor pack_ph3_image_latent(const sd::Tensor& image, + const sd::Tensor& previous = {}) { + auto video = image.dim() == 4 ? image.unsqueeze(2) : image; + GGML_ASSERT(video.dim() == 5 && video.shape()[2] == 1 && video.shape()[3] == 24 && video.shape()[4] == 1); + const int64_t spatial = video.shape()[0] * video.shape()[1]; + // The image DiT still attends to two stereo audio latents (32 channels). + auto shape = video.shape(); + shape[3] += (128 + spatial - 1) / spatial; + auto packed = sd::zeros(shape); + std::copy_n(video.data(), video.numel(), packed.data()); + if (!previous.empty()) { + const int64_t offset = previous.shape()[0] * previous.shape()[1] * 24; + GGML_ASSERT(previous.numel() >= offset + 128); + std::copy_n(previous.data() + offset, 128, packed.data() + video.numel()); + } + return packed; + } + static std::optional prepare_image_generation_latents(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params, GenerationRequest* request, @@ -323,7 +351,10 @@ namespace sd::pipeline { continue; } sd::Tensor ref_latent; - if (ref_image_params.resize_before_vae && !sd_version_is_pid(sd->version)) { + if (sd->version == VERSION_P_H3_IMAGE) { + auto resized = resize_ph3_reference(ref_images[i], int64_t(request->width) * request->height); + ref_latent = sd->encode_first_stage(resized); + } else if (ref_image_params.resize_before_vae && !sd_version_is_pid(sd->version)) { LOG_VERBOSE("auto resize ref images"); double vae_width; double vae_height; @@ -450,6 +481,16 @@ namespace sd::pipeline { } ImageGenerationLatents latents; + if (sd->version == VERSION_P_H3_IMAGE) { + init_latent = pack_ph3_image_latent(init_latent); + latent_mask = latent_mask.unsqueeze(2); + for (size_t i = 0; i < ref_latents.size(); ++i) { + latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::IMAGE, static_cast(i), -1}); + } + for (auto& image : ref_images) { + image = resize_ph3_reference(image, 1000000); + } + } latents.init_latent = std::move(init_latent); latents.concat_latent = std::move(concat_latent); latents.img_uncond_concat_latent = std::move(img_uncond_concat_latent); @@ -502,6 +543,9 @@ namespace sd::pipeline { if (cond.c_concat.empty() && ref_image_params.pass_to_dit) { cond.c_concat = latents->concat_latent; // TODO: optimize } + if (sd->version == VERSION_P_H3_IMAGE && ref_image_params.pass_to_dit) { + cond.c_reference_blocks = latents->minimax_reference_blocks; + } bool use_ref_latent_img_cfg = request->use_img_uncond && !latents->ref_images.empty() && @@ -546,6 +590,9 @@ namespace sd::pipeline { if (uncond.c_concat.empty() && ref_image_params.pass_to_dit) { uncond.c_concat = latents->concat_latent; // TODO: optimize } + if (sd->version == VERSION_P_H3_IMAGE && ref_image_params.pass_to_dit) { + uncond.c_reference_blocks = latents->minimax_reference_blocks; + } } SDCondition img_uncond; @@ -1047,13 +1094,19 @@ namespace sd::pipeline { sd->rng->manual_seed(cur_seed); sd->sampler_rng->manual_seed(cur_seed); + auto hires_source = sd->version == VERSION_P_H3_IMAGE + ? sd::ops::slice(final_latents[b], 3, 0, 24) + : final_latents[b]; sd::Tensor upscaled = upscale_hires_latent(sd, - final_latents[b], + hires_source, request, hires_upscaler.get()); if (upscaled.empty()) { return false; } + if (sd->version == VERSION_P_H3_IMAGE) { + upscaled = pack_ph3_image_latent(upscaled, final_latents[b]); + } sd::Tensor noise = sd::randn_like(upscaled, sd->rng); diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp index 8b85f9b46b..27f6f06ea9 100644 --- a/src/pipeline/model_builders.cpp +++ b/src/pipeline/model_builders.cpp @@ -159,6 +159,13 @@ namespace sd::model_builders { version, weight_manager); } else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::MINIMAX_H3) { + if (version == VERSION_P_H3_IMAGE) { + // The video VAE also loads successfully, but uses different temporal decoding and posterior sampling. + auto proj = tensor_storage_map.find("first_stage_model.decoder.proj_out.weight"); + if (proj == tensor_storage_map.end() || proj->second.ne[1] != 768) { + throw std::runtime_error("P-H3-Image requires its single-frame VAE; pass --vae p-h3-image_vae_bf16.safetensors"); + } + } return std::make_shared(ctx.backends.runtime_backend(SDBackendModule::VAE), tensor_storage_map, "first_stage_model", From 3aa920ddeed9bcab365b91c94146698a9e102304 Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 4 Nov 2026 22:59:24 +0800 Subject: [PATCH 2/2] fix: return load failure on VAE construction errors --- src/pipeline/model_builders.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp index 27f6f06ea9..e78a0d666e 100644 --- a/src/pipeline/model_builders.cpp +++ b/src/pipeline/model_builders.cpp @@ -110,7 +110,7 @@ namespace sd::model_builders { return false; } - bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) { + bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) try { const auto* sd_ctx_params = &ctx.params; const auto& tensor_storage_map = ctx.tensor_storage_map; const auto version = ctx.version; @@ -247,6 +247,9 @@ namespace sd::model_builders { } runners = std::move(result); return true; + } catch (const std::exception& error) { + LOG_ERROR("failed to build VAE runners: %s", error.what()); + return false; } bool build_control_net_runner(const Context& ctx, std::shared_ptr& runner) {