From dc857537cee67c7980d10cda1b229f3d8ccc53ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sat, 11 Jul 2026 18:48:27 +0200 Subject: [PATCH 01/13] prepare edit params --- examples/common/common.cpp | 6 +++ examples/common/common.h | 2 + include/stable-diffusion.h | 1 + src/model/diffusion/model.hpp | 18 +++++++ src/stable-diffusion.cpp | 99 ++++++++++++++++++++++++++++++++--- 5 files changed, 119 insertions(+), 7 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 1dfb6aa70..e9f7f85f1 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -975,6 +975,11 @@ ArgOptions SDGenerationParams::get_options() { "extra VAE tiling args, key=value list. LTX video VAE supports temporal_tile_frames (default: 4), temporal_tile_overlap (default: 1)", (int)',', &extra_tiling_args}, + {"", + "--ref-image-mode", + "Key-value list to set up the way the reference images are processed (empty = auto-detect from model weigths)", + (int)',', + &ref_mode}, }; options.int_options = { @@ -2428,6 +2433,7 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { params.ref_images_count = static_cast(ref_image_views.size()); params.auto_resize_ref_image = auto_resize_ref_image; params.increase_ref_index = increase_ref_index; + params.ref_image_mode = ref_mode.c_str(); params.mask_image = mask_image.get(); params.width = get_resolved_width(); params.height = get_resolved_height(); diff --git a/examples/common/common.h b/examples/common/common.h index 3a5b107bc..ac98e5b22 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -226,6 +226,8 @@ struct SDGenerationParams { float vace_strength = 1.f; sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; std::string extra_tiling_args; + + std::string ref_mode; std::string pm_id_images_dir; std::string pm_id_embed_path; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index eeb59f873..3bfd92a49 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -365,6 +365,7 @@ typedef struct { int ref_images_count; bool auto_resize_ref_image; bool increase_ref_index; + const char* ref_image_mode; sd_image_t mask_image; int width; int height; diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index 8ef00023e..70cf127c3 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -10,6 +10,23 @@ #include "model/common/rope.hpp" #include "model_manager.h" +struct EditModeParams { + bool use_VLM = false; + bool use_dit_refs = true; + Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED; + bool force_timestep_0 = false; +}; + +const std::unordered_map REF_PRESETS = { + // {preset_name, {VLM, refs, mode, t_0 }}, + {"flux_kontext", {false, true, Rope::RefIndexMode::FIXED, false}}, + {"flux2", {false, true, Rope::RefIndexMode::INCREASE, false}}, + {"qwen", {true, true, Rope::RefIndexMode::INCREASE, false}}, + {"qwen_layered", {true, true, Rope::RefIndexMode::DECREASE, false}}, + {"z_image_omni", {true, true, Rope::RefIndexMode::FIXED, false}}, + {"krea2_ostris_edit", {true, true, Rope::RefIndexMode::INCREASE, true }}, +}; + struct UNetDiffusionExtra { int num_video_frames = -1; const std::vector>* controls = nullptr; @@ -74,6 +91,7 @@ struct DiffusionParams { const sd::Tensor* c_concat = nullptr; const sd::Tensor* y = nullptr; const std::vector>* ref_latents = nullptr; + EditModeParams edit_params = {false, false, Rope::RefIndexMode::FIXED, false}; Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED; DiffusionExtraParams extra = std::monostate{}; }; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 523bc9d01..4b7eb9d05 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2298,6 +2298,7 @@ class StableDiffusionGGML { const std::vector& sigmas, const std::vector>& ref_latents, bool increase_ref_index, + EditModeParams& edit_params, const sd::Tensor& denoise_mask, const sd::Tensor& vace_context, float vace_strength, @@ -2461,6 +2462,7 @@ class StableDiffusionGGML { diffusion_params.x = &noised_input; diffusion_params.timesteps = ×teps_tensor; diffusion_params.ref_index_mode = Rope::ref_index_mode_from_bool(increase_ref_index); + diffusion_params.edit_params = edit_params; sd::guidance::GuidanceInput step_guidance_input; step_guidance_input.step = step; step_guidance_input.schedule_size = sigmas.size(); @@ -2840,6 +2842,73 @@ class StableDiffusionGGML { auto flow_denoiser = std::dynamic_pointer_cast(denoiser); return !!flow_denoiser; } + + std::string get_default_preset_for_version(SDVersion version) { + if (sd_version_is_flux(version) || sd_version_is_longcat(version)) { + return "flux_kontext"; + } else if (sd_version_is_flux2(version) || sd_version_is_sefi_image(version)) { + return "flux2"; + } else if (version == VERSION_QWEN_IMAGE_LAYERED){ + return "qwen_layered"; + } else if (sd_version_is_qwen_image(version)) { + return "qwen"; + } else if (sd_version_is_z_image(version)){ + return "z_image_omni"; + } else if (sd_version_is_krea2(version)){ + // have to make a choice between "qwen" mode (for lbouaraba/krea2edit) + // and "krea2_ostris_edit" (for ostris edit) + // since krea2 ostris edit support predates, it should probably be default + return "krea2_ostris_edit"; + } + return "default"; + } + + void set_edit_mode_params(EditModeParams& params, std::string overrides) { + std::string preset_name = get_default_preset_for_version(version); + + for (const auto& [key, value] : parse_key_value_args(overrides, "edit mode args")) { + if (key == "preset") { + std::string requested_preset_name = value; + if (REF_PRESETS.count(requested_preset_name)) { + preset_name = requested_preset_name; + } else if (value != "default") { + std::string valid_list; + for (auto const& [name, _] : REF_PRESETS) { + valid_list += (valid_list.empty() ? "" : ", ") + name; + } + LOG_WARN("ignoring invalid edit mode preset '%s'. Valid options: [%s]", value.c_str(), valid_list.c_str()); + } + break; + } + } + if (preset_name != "default") { + params = REF_PRESETS.at(preset_name); + } + + for (const auto& [key, value] : parse_key_value_args(overrides, "edit mode args")) { + if (key == "use_vlm") { + if (!parse_strict_bool(value, params.use_VLM)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "pass_to_dit") { + if (!parse_strict_bool(value, params.use_dit_refs)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "ref_index_mode") { + if (value == "fixed"){ + params.ref_index_mode = Rope::RefIndexMode::FIXED; + } else if (value == "increase"){ + params.ref_index_mode = Rope::RefIndexMode::INCREASE; + } else if (value == "decrease"){ + params.ref_index_mode = Rope::RefIndexMode::DECREASE; + } + } else if (key == "force_timestep_0"){ + if (!parse_strict_bool(value, params.force_timestep_0)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } + } + } }; /*================================================= SD API ==================================================*/ @@ -4703,7 +4772,8 @@ static std::optional prepare_image_generation_embeds(sd_c const sd_img_gen_params_t* sd_img_gen_params, GenerationRequest* request, SamplePlan* plan, - ImageGenerationLatents* latents) { + ImageGenerationLatents* latents, + EditModeParams* edit_params) { ConditionerRunnerDoneOnExit conditioner_runner_done{sd_ctx->sd->cond_stage_model.get()}; ConditionerParams condition_params; @@ -4711,7 +4781,10 @@ static std::optional prepare_image_generation_embeds(sd_c condition_params.clip_skip = request->clip_skip; condition_params.width = request->width; condition_params.height = request->height; - condition_params.ref_images = &latents->ref_images; + if(edit_params->use_VLM){ + condition_params.ref_images = &latents->ref_images; + } + sd_ctx->sd->prepare_generation_extensions(request->pm_params, request->pulid_params, @@ -4721,7 +4794,7 @@ static std::optional prepare_image_generation_embeds(sd_c condition_params.zero_out_masked = false; auto cond = sd_ctx->sd->cond_stage_model->get_learned_condition(sd_ctx->sd->n_threads, condition_params); - if (cond.c_concat.empty()) { + if (cond.c_concat.empty() && edit_params->use_dit_refs) { cond.c_concat = latents->concat_latent; // TODO: optimize } @@ -4750,7 +4823,7 @@ static std::optional prepare_image_generation_embeds(sd_c uncond = sd_ctx->sd->cond_stage_model->get_learned_condition(sd_ctx->sd->n_threads, condition_params); } - if (uncond.c_concat.empty()) { + if (uncond.c_concat.empty() && edit_params->use_dit_refs) { uncond.c_concat = latents->concat_latent; // TODO: optimize } } @@ -4774,7 +4847,7 @@ static std::optional prepare_image_generation_embeds(sd_c } img_uncond = sd_ctx->sd->cond_stage_model->get_learned_condition(sd_ctx->sd->n_threads, condition_params); - if (img_uncond.c_concat.empty()) { + if (img_uncond.c_concat.empty() && edit_params->use_dit_refs) { img_uncond.c_concat = latents->img_uncond_concat_latent; // TODO: optimize } } @@ -5091,6 +5164,10 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, sd_ctx->sd->apply_loras(sd_img_gen_params->loras, sd_img_gen_params->lora_count); apply_circular_axes_to_diffusion(sd_ctx, sd_img_gen_params->circular_x, sd_img_gen_params->circular_y); + EditModeParams edit_params; + // TODO: parse EditModeParams from sd_img_gen_params->ref_image_mode + sd_ctx->sd->set_edit_mode_params(edit_params, sd_img_gen_params->ref_image_mode); + ImageVaeAxesGuard axes_guard(sd_ctx, sd_img_gen_params, request); SamplePlan plan(sd_ctx, sd_img_gen_params, request); @@ -5107,7 +5184,8 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, sd_img_gen_params, &request, &plan, - &latents); + &latents, + &edit_params); if (!embeds_opt.has_value()) { return false; } @@ -5154,6 +5232,7 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, plan.sigmas, latents.ref_latents, request.increase_ref_index, + edit_params, latents.denoise_mask, sd::Tensor(), 1.f, @@ -5275,6 +5354,7 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, hires_sigma_sched, latents.ref_latents, request.increase_ref_index, + edit_params, hires_denoise_mask, sd::Tensor(), 1.f, @@ -5945,6 +6025,8 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, sd_ctx->sd->reset_cancel_flag(); + EditModeParams edit_params; + if (num_frames_out != nullptr) { *num_frames_out = 0; } @@ -6035,6 +6117,7 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, high_noise_sigmas, std::vector>{}, false, + edit_params, latents.denoise_mask, latents.vace_context, request.vace_strength, @@ -6077,6 +6160,7 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, plan.sigmas, std::vector>{}, false, + edit_params, latents.denoise_mask, latents.vace_context, request.vace_strength, @@ -6202,7 +6286,7 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, x_t, std::move(noise), embeds.cond, - hires_request.use_uncond ? embeds.uncond : SDCondition(), + hires_request.use_uncond ? embeds.uncond : SDCondition(), embeds.img_uncond, sd::Tensor(), 0.f, @@ -6215,6 +6299,7 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, hires_sigma_sched, std::vector>{}, false, + edit_params, hires_denoise_mask, sd::Tensor(), hires_request.vace_strength, From e72793f0b553f3e12ce28f7cdf105867f4514187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sat, 11 Jul 2026 18:49:31 +0200 Subject: [PATCH 02/13] support edit params for krea2 --- src/model/diffusion/krea2.hpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/model/diffusion/krea2.hpp b/src/model/diffusion/krea2.hpp index 8ca0bb51f..251ab3216 100644 --- a/src/model/diffusion/krea2.hpp +++ b/src/model/diffusion/krea2.hpp @@ -615,7 +615,8 @@ namespace Krea2 { ggml_tensor* timestep, ggml_tensor* context, ggml_tensor* pe, - std::vector ref_latents = {}) { + std::vector ref_latents = {}, + bool zero_timestep_refs = false) { int64_t W = x->ne[0]; int64_t H = x->ne[1]; int64_t N = x->ne[3]; @@ -645,7 +646,7 @@ namespace Krea2 { auto tvec = tproj->forward(ctx, t); ggml_tensor* tvec_0 = nullptr; - if (ref_latents.size() > 0) { + if (ref_latents.size() > 0 && zero_timestep_refs) { // "index_timestep_zero" mode: use timestep = 0 for ref latents auto timestep_0 = ggml_scale(ctx->ggml_ctx, timestep, 0.0f); auto t_0 = ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep_0, static_cast(config.timestep_dim), 10000, 1000.f); @@ -719,7 +720,7 @@ namespace Krea2 { const sd::Tensor& timesteps_tensor, const sd::Tensor& context_tensor, const std::vector>& ref_latents_tensor = {}, - Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED) { + EditModeParams edit_params = REF_PRESETS.at("krea2_ostris_edit")) { ggml_cgraph* gf = new_graph_custom(KREA2_GRAPH_SIZE); ggml_tensor* x = make_input(x_tensor); ggml_tensor* timesteps = make_input(timesteps_tensor); @@ -741,13 +742,13 @@ namespace Krea2 { config.theta, config.axes_dim, ref_latents, - ref_index_mode); + edit_params.ref_index_mode); int pos_len = static_cast(pe_vec.size() / config.axes_dim_sum / 2); auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len); set_backend_tensor_data(pe, pe_vec.data()); auto runner_ctx = get_context(); - ggml_tensor* out = model.forward(&runner_ctx, x, timesteps, context, pe, ref_latents); + ggml_tensor* out = model.forward(&runner_ctx, x, timesteps, context, pe, ref_latents, edit_params.force_timestep_0); ggml_build_forward_expand(gf, out); return gf; } @@ -757,9 +758,9 @@ namespace Krea2 { const sd::Tensor& timesteps, const sd::Tensor& context, const std::vector>& ref_latents = {}, - Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED) { + EditModeParams edit_params = REF_PRESETS.at("krea2_ostris_edit")) { auto get_graph = [&]() -> ggml_cgraph* { - return build_graph(x, timesteps, context, ref_latents, ref_index_mode); + return build_graph(x, timesteps, context, ref_latents, edit_params); }; return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } @@ -773,8 +774,8 @@ namespace Krea2 { *diffusion_params.x, *diffusion_params.timesteps, tensor_or_empty(diffusion_params.context), - diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, - diffusion_params.ref_index_mode); + diffusion_params.ref_latents && diffusion_params.edit_params.use_dit_refs ? *diffusion_params.ref_latents : empty_ref_latents, + diffusion_params.edit_params); } }; } // namespace Krea2 From 78b1fc2e042b8ac5d4aac734500cc84c082c3dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sat, 11 Jul 2026 20:19:22 +0200 Subject: [PATCH 03/13] add image resize options --- src/conditioning/conditioner.hpp | 198 +++++++++++++++++++++++-------- src/model/diffusion/model.hpp | 33 ++++-- src/stable-diffusion.cpp | 64 +++++++--- 3 files changed, 218 insertions(+), 77 deletions(-) diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 61b5791c9..3ea58378a 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -7,6 +7,7 @@ #include "core/tensor_ggml.hpp" #include "core/util.h" +#include "model/diffusion/model.hpp" #include "model/te/clip.hpp" #include "model/te/llm.hpp" #include "model/te/t5.hpp" @@ -106,6 +107,7 @@ struct ConditionerParams { int height = -1; bool zero_out_masked = false; const std::vector>* ref_images = nullptr; // for qwen image edit + EditModeParams edit_mode; }; struct Conditioner { @@ -1993,6 +1995,54 @@ struct LLMEmbedder : public Conditioner { return new_hidden_states; } + void resize_image_dims(int height, int width, int& h_bar, int& w_bar, int factor, int min_size, int max_size, CondResizeMode mode) { + if (min_size > 0 && min_size == max_size) { + if (mode == CondResizeMode::AREA) { + double beta = std::sqrt(static_cast(min_size) / (static_cast(height) * width)); + h_bar = std::max(static_cast(factor), + static_cast(std::round(height * beta / factor)) * static_cast(factor)); + w_bar = std::max(static_cast(factor), + static_cast(std::round(width * beta / factor)) * static_cast(factor)); + } else if (mode == CondResizeMode::LONGEST_SIDE) { + int current_max_side = std::max(height, width); + double beta = static_cast(min_size) / current_max_side; + h_bar = std::max(static_cast(factor), + static_cast(std::round(height * beta / factor)) * static_cast(factor)); + w_bar = std::max(static_cast(factor), + static_cast(std::round(width * beta / factor)) * static_cast(factor)); + } + return; + } + + if (mode == CondResizeMode::AREA) { + double current_area = static_cast(h_bar) * w_bar; + if (max_size > 0 && current_area > max_size) { + double beta = std::sqrt((static_cast(height) * width) / static_cast(max_size)); + h_bar = std::max(static_cast(factor), + static_cast(std::floor(height / beta / factor)) * static_cast(factor)); + w_bar = std::max(static_cast(factor), + static_cast(std::floor(width / beta / factor)) * static_cast(factor)); + } else if (min_size > 0 && current_area < min_size) { + double beta = std::sqrt(static_cast(min_size) / (static_cast(height) * width)); + h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); + w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); + } + } else if (mode == CondResizeMode::LONGEST_SIDE) { + int current_max_side = std::max(height, width); + if (max_size > 0 && current_max_side > max_size) { + double beta = static_cast(max_size) / current_max_side; + h_bar = std::max(static_cast(factor), + static_cast(std::floor(height / beta / factor)) * static_cast(factor)); + w_bar = std::max(static_cast(factor), + static_cast(std::floor(width / beta / factor)) * static_cast(factor)); + } else if (min_size > 0 && current_max_side < min_size) { + double beta = static_cast(min_size) / current_max_side; + h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); + w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); + } + } + } + SDCondition get_learned_condition(int n_threads, const ConditionerParams& conditioner_params) override { std::string prompt; @@ -2008,6 +2058,7 @@ struct LLMEmbedder : public Conditioner { std::set out_layers; int64_t t0 = ggml_time_ms(); + CondResizeMode resize_mode = conditioner_params.edit_mode.cond_refs_resize_mode; if (sd_version_is_lingbot_video(version)) { const int pad_token = 151643; @@ -2043,25 +2094,32 @@ struct LLMEmbedder : public Conditioner { double factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size; int height = static_cast(image.shape()[1]); int width = static_cast(image.shape()[0]); - int min_pixels = static_cast(4 * factor * factor); - int max_pixels = static_cast(16384 * factor * factor); - int h_bar = std::max(static_cast(factor), static_cast(std::round(height / factor) * factor)); - int w_bar = std::max(static_cast(factor), static_cast(std::round(width / factor) * factor)); + + int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + if (min_pixels <= 0) { + if (resize_mode == CondResizeMode::AREA) { + min_pixels = static_cast(4 * factor * factor); + } else { + min_pixels = static_cast(2 * factor); + } + } + int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + if (max_pixels <= 0) { + if (resize_mode == CondResizeMode::AREA) { + max_pixels = static_cast(16384 * factor * factor); + } else { + max_pixels = static_cast(128 * factor); + } + } + + int h_bar = std::max(static_cast(factor), static_cast(std::round(height / factor) * factor)); + int w_bar = std::max(static_cast(factor), static_cast(std::round(width / factor) * factor)); if (std::max(height, width) > 200 * std::min(height, width)) { LOG_WARN("LingBotVideo image aspect ratio is very large: %dx%d", width, height); } - if (h_bar * w_bar > max_pixels) { - double beta = std::sqrt((height * width) / static_cast(max_pixels)); - h_bar = std::max(static_cast(factor), - static_cast(std::floor(height / beta / factor)) * static_cast(factor)); - w_bar = std::max(static_cast(factor), - static_cast(std::floor(width / beta / factor)) * static_cast(factor)); - } else if (h_bar * w_bar < min_pixels) { - double beta = std::sqrt(static_cast(min_pixels) / (height * width)); - h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); - w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); - } + + resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode); LOG_DEBUG("resize LingBotVideo ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); auto resized_image = clip_preprocess(image, w_bar, h_bar); @@ -2092,8 +2150,21 @@ struct LLMEmbedder : public Conditioner { prompt_template_encode_start_idx = 64; int image_embed_idx = 64 + 6; - int min_pixels = 384 * 384; - int max_pixels = 560 * 560; + int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + if (min_pixels <= 0) { + min_pixels = 384; + if (resize_mode == CondResizeMode::AREA) { + min_pixels *= min_pixels; + } + } + int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + if (max_pixels <= 0) { + max_pixels = 560; + if (resize_mode == CondResizeMode::AREA) { + max_pixels *= max_pixels; + } + } + std::string placeholder = "<|image_pad|>"; std::string img_prompt; @@ -2105,17 +2176,7 @@ struct LLMEmbedder : public Conditioner { int h_bar = static_cast(std::round(height / factor) * factor); int w_bar = static_cast(std::round(width / factor) * factor); - if (static_cast(h_bar) * w_bar > max_pixels) { - double beta = std::sqrt((height * width) / static_cast(max_pixels)); - h_bar = std::max(static_cast(factor), - static_cast(std::floor(height / beta / factor)) * static_cast(factor)); - w_bar = std::max(static_cast(factor), - static_cast(std::floor(width / beta / factor)) * static_cast(factor)); - } else if (static_cast(h_bar) * w_bar < min_pixels) { - double beta = std::sqrt(static_cast(min_pixels) / (height * width)); - h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); - w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); - } + resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode); LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); @@ -2170,16 +2231,33 @@ struct LLMEmbedder : public Conditioner { std::string img_prompt; const std::string placeholder = "<|image_pad|>"; + int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + if (min_pixels <= 0) { + min_pixels = 384; + if (resize_mode == CondResizeMode::AREA) { + min_pixels *= min_pixels; + } + } + int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + if (max_pixels <= 0) { + max_pixels = 384; + if (resize_mode == CondResizeMode::AREA) { + max_pixels *= max_pixels; + } + } + for (int i = 0; i < conditioner_params.ref_images->size(); i++) { const auto& image = (*conditioner_params.ref_images)[i]; double factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size; int height = static_cast(image.shape()[1]); int width = static_cast(image.shape()[0]); - double beta = std::sqrt((384.0 * 384.0) / (static_cast(height) * static_cast(width))); - int h_bar = std::max(static_cast(factor), - static_cast(std::round(height * beta / factor)) * static_cast(factor)); - int w_bar = std::max(static_cast(factor), - static_cast(std::round(width * beta / factor)) * static_cast(factor)); + + int h_bar = std::max(static_cast(factor), + static_cast(std::round(height / factor)) * static_cast(factor)); + int w_bar = std::max(static_cast(factor), + static_cast(std::round(width / factor)) * static_cast(factor)); + + resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode); LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); @@ -2221,17 +2299,32 @@ struct LLMEmbedder : public Conditioner { if (llm->enable_vision && conditioner_params.ref_images != nullptr && !conditioner_params.ref_images->empty()) { std::string img_prompt = ""; const std::string placeholder = "<|image_pad|>"; + int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + if (min_pixels <= 0) { + min_pixels = 384; + if (resize_mode == CondResizeMode::AREA) { + min_pixels *= min_pixels; + } + } + int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + if (max_pixels <= 0) { + max_pixels = 1024; + if (resize_mode == CondResizeMode::AREA) { + max_pixels *= max_pixels; + } + } for (int i = 0; i < conditioner_params.ref_images->size(); i++) { const auto& image = (*conditioner_params.ref_images)[i]; double factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size; int height = static_cast(image.shape()[1]); int width = static_cast(image.shape()[0]); - double beta = std::sqrt((384.0 * 384.0) / (static_cast(height) * static_cast(width))); - int h_bar = std::max(static_cast(factor), - static_cast(std::round(height * beta / factor)) * static_cast(factor)); - int w_bar = std::max(static_cast(factor), - static_cast(std::round(width * beta / factor)) * static_cast(factor)); + + int h_bar = std::max(static_cast(factor), + static_cast(std::round(height / factor)) * static_cast(factor)); + int w_bar = std::max(static_cast(factor), + static_cast(std::round(width / factor)) * static_cast(factor)); + resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode); LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); @@ -2268,8 +2361,21 @@ struct LLMEmbedder : public Conditioner { min_length = 512 + prompt_template_encode_start_idx; int image_embed_idx = 36 + 6; - int min_pixels = 384 * 384; - int max_pixels = 560 * 560; + int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + if (min_pixels <= 0) { + min_pixels = 384; + if (resize_mode == CondResizeMode::AREA) { + min_pixels *= min_pixels; + } + } + int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + if (max_pixels <= 0) { + max_pixels = 560; + if (resize_mode == CondResizeMode::AREA) { + max_pixels *= max_pixels; + } + } + std::string placeholder = "<|image_pad|>"; std::string img_prompt; @@ -2281,17 +2387,7 @@ struct LLMEmbedder : public Conditioner { int h_bar = static_cast(std::round(height / factor) * factor); int w_bar = static_cast(std::round(width / factor) * factor); - if (static_cast(h_bar) * w_bar > max_pixels) { - double beta = std::sqrt((height * width) / static_cast(max_pixels)); - h_bar = std::max(static_cast(factor), - static_cast(std::floor(height / beta / factor)) * static_cast(factor)); - w_bar = std::max(static_cast(factor), - static_cast(std::floor(width / beta / factor)) * static_cast(factor)); - } else if (static_cast(h_bar) * w_bar < min_pixels) { - double beta = std::sqrt(static_cast(min_pixels) / (height * width)); - h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); - w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); - } + resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode); LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index 70cf127c3..4354aa8fe 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -10,21 +10,32 @@ #include "model/common/rope.hpp" #include "model_manager.h" +enum class CondResizeMode { + NONE, + LONGEST_SIDE, + AREA, +}; + struct EditModeParams { - bool use_VLM = false; - bool use_dit_refs = true; - Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED; - bool force_timestep_0 = false; + bool use_VLM = false; + bool use_dit_refs = true; + Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED; + bool force_timestep_0 = false; + bool resize_vae_refs = true; + int vae_refs_size = -1; + CondResizeMode cond_refs_resize_mode = CondResizeMode::LONGEST_SIDE; + int cond_refs_max_size = -1; + int cond_refs_min_size = -1; }; const std::unordered_map REF_PRESETS = { - // {preset_name, {VLM, refs, mode, t_0 }}, - {"flux_kontext", {false, true, Rope::RefIndexMode::FIXED, false}}, - {"flux2", {false, true, Rope::RefIndexMode::INCREASE, false}}, - {"qwen", {true, true, Rope::RefIndexMode::INCREASE, false}}, - {"qwen_layered", {true, true, Rope::RefIndexMode::DECREASE, false}}, - {"z_image_omni", {true, true, Rope::RefIndexMode::FIXED, false}}, - {"krea2_ostris_edit", {true, true, Rope::RefIndexMode::INCREASE, true }}, + {"flux_kontext", {false, true, Rope::RefIndexMode::FIXED, false, true, -1, CondResizeMode::NONE, -1, -1}}, + {"flux2", {false, true, Rope::RefIndexMode::INCREASE, false, true, -1, CondResizeMode::NONE, -1, -1}}, + {"qwen", {true, true, Rope::RefIndexMode::INCREASE, false, true, -1, CondResizeMode::AREA, -1, -1}}, + {"qwen_layered", {true, true, Rope::RefIndexMode::DECREASE, false, true, -1, CondResizeMode::AREA, -1, -1}}, + {"z_image_omni", {true, true, Rope::RefIndexMode::FIXED, false, true, -1, CondResizeMode::AREA, -1, -1}}, + {"krea2_ostris_edit", {true, true, Rope::RefIndexMode::INCREASE, true, true, -1, CondResizeMode::AREA, -1, -1}}, + {"krea2_edit", {true, true, Rope::RefIndexMode::INCREASE, false, true, -1, CondResizeMode::LONGEST_SIDE, 768, 768}}, }; struct UNetDiffusionExtra { diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 4b7eb9d05..7df7f8628 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2855,8 +2855,8 @@ class StableDiffusionGGML { } else if (sd_version_is_z_image(version)){ return "z_image_omni"; } else if (sd_version_is_krea2(version)){ - // have to make a choice between "qwen" mode (for lbouaraba/krea2edit) - // and "krea2_ostris_edit" (for ostris edit) + // have to make a choice between "krea2_edit" mode (for lbouaraba/krea2edit) + // and "krea2_ostris_edit" (for krea2 ostris edit) // since krea2 ostris edit support predates, it should probably be default return "krea2_ostris_edit"; } @@ -2884,28 +2884,59 @@ class StableDiffusionGGML { if (preset_name != "default") { params = REF_PRESETS.at(preset_name); } - + for (const auto& [key, value] : parse_key_value_args(overrides, "edit mode args")) { if (key == "use_vlm") { - if (!parse_strict_bool(value, params.use_VLM)) { + if (!parse_strict_bool(value, params.use_VLM)) { LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); } } else if (key == "pass_to_dit") { - if (!parse_strict_bool(value, params.use_dit_refs)) { + if (!parse_strict_bool(value, params.use_dit_refs)) { LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); } } else if (key == "ref_index_mode") { - if (value == "fixed"){ + if (value == "fixed") { params.ref_index_mode = Rope::RefIndexMode::FIXED; - } else if (value == "increase"){ + } else if (value == "increase") { params.ref_index_mode = Rope::RefIndexMode::INCREASE; - } else if (value == "decrease"){ + } else if (value == "decrease") { params.ref_index_mode = Rope::RefIndexMode::DECREASE; - } - } else if (key == "force_timestep_0"){ - if (!parse_strict_bool(value, params.force_timestep_0)) { + } else { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "force_timestep_0") { + if (!parse_strict_bool(value, params.force_timestep_0)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "resize_vae_refs") { + if (!parse_strict_bool(value, params.resize_vae_refs)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "vae_refs_size") { + if (!parse_strict_int(value, params.vae_refs_size)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "cond_refs_resize_mode") { + if (value == "longest_side") { + params.cond_refs_resize_mode = CondResizeMode::LONGEST_SIDE; + } else if (value == "area") { + params.cond_refs_resize_mode = CondResizeMode::AREA; + } else if (value == "none") { + params.cond_refs_resize_mode = CondResizeMode::NONE; + } else { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "cond_refs_max_size") { + if (!parse_strict_int(value, params.cond_refs_max_size)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } + } else if (key == "cond_refs_min_size") { + if (!parse_strict_int(value, params.cond_refs_min_size)) { LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); } + } else if (key != "preset") { + // Optional: catch-all for unknown keys + LOG_WARN("ignoring unknown edit mode arg '%s'", key.c_str()); } } } @@ -4500,7 +4531,8 @@ static sd::Tensor ensure_image_tensor_channels(sd::Tensor image, i static std::optional prepare_image_generation_latents(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* sd_img_gen_params, GenerationRequest* request, - SamplePlan* plan) { + SamplePlan* plan, + EditModeParams* edit_params) { int64_t prepare_start_ms = ggml_time_ms(); sd::Tensor init_image_tensor; @@ -4643,9 +4675,10 @@ static std::optional prepare_image_generation_latents(sd continue; } sd::Tensor ref_latent; - if (request->auto_resize_ref_image && !sd_version_is_pid(sd_ctx->sd->version)) { + if (edit_params->resize_vae_refs && !sd_version_is_pid(sd_ctx->sd->version)) { LOG_DEBUG("auto resize ref images"); - int vae_image_size = std::min(1024 * 1024, request->width * request->height); + int target_pixels = edit_params->vae_refs_size > 0 ? edit_params->vae_refs_size : 1024 * 1024; + int vae_image_size = std::min(target_pixels, request->width * request->height); double vae_width = sqrt(vae_image_size * ref_images[i].shape()[0] / ref_images[i].shape()[1]); double vae_height = vae_width * ref_images[i].shape()[1] / ref_images[i].shape()[0]; @@ -5174,7 +5207,8 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, auto latents_opt = prepare_image_generation_latents(sd_ctx, sd_img_gen_params, &request, - &plan); + &plan, + &edit_params); if (!latents_opt.has_value()) { return false; } From a9daa77e254ef7f817257ed3a7b3ecce07a38b51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sat, 11 Jul 2026 21:17:56 +0200 Subject: [PATCH 04/13] fixes & refactors --- src/conditioning/conditioner.hpp | 25 +++++++++++++------------ src/model/diffusion/model.hpp | 2 +- src/stable-diffusion.cpp | 26 ++++++++++++++++++++++---- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 3ea58378a..db582729f 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -107,7 +107,7 @@ struct ConditionerParams { int height = -1; bool zero_out_masked = false; const std::vector>* ref_images = nullptr; // for qwen image edit - EditModeParams edit_mode; + EditModeParams edit_params; }; struct Conditioner { @@ -2058,7 +2058,7 @@ struct LLMEmbedder : public Conditioner { std::set out_layers; int64_t t0 = ggml_time_ms(); - CondResizeMode resize_mode = conditioner_params.edit_mode.cond_refs_resize_mode; + CondResizeMode resize_mode = conditioner_params.edit_params.cond_refs_resize_mode; if (sd_version_is_lingbot_video(version)) { const int pad_token = 151643; @@ -2095,7 +2095,7 @@ struct LLMEmbedder : public Conditioner { int height = static_cast(image.shape()[1]); int width = static_cast(image.shape()[0]); - int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + int min_pixels = conditioner_params.edit_params.cond_refs_min_size; if (min_pixels <= 0) { if (resize_mode == CondResizeMode::AREA) { min_pixels = static_cast(4 * factor * factor); @@ -2103,7 +2103,7 @@ struct LLMEmbedder : public Conditioner { min_pixels = static_cast(2 * factor); } } - int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + int max_pixels = conditioner_params.edit_params.cond_refs_max_size; if (max_pixels <= 0) { if (resize_mode == CondResizeMode::AREA) { max_pixels = static_cast(16384 * factor * factor); @@ -2150,14 +2150,14 @@ struct LLMEmbedder : public Conditioner { prompt_template_encode_start_idx = 64; int image_embed_idx = 64 + 6; - int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + int min_pixels = conditioner_params.edit_params.cond_refs_min_size; if (min_pixels <= 0) { min_pixels = 384; if (resize_mode == CondResizeMode::AREA) { min_pixels *= min_pixels; } } - int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + int max_pixels = conditioner_params.edit_params.cond_refs_max_size; if (max_pixels <= 0) { max_pixels = 560; if (resize_mode == CondResizeMode::AREA) { @@ -2231,14 +2231,14 @@ struct LLMEmbedder : public Conditioner { std::string img_prompt; const std::string placeholder = "<|image_pad|>"; - int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + int min_pixels = conditioner_params.edit_params.cond_refs_min_size; if (min_pixels <= 0) { min_pixels = 384; if (resize_mode == CondResizeMode::AREA) { min_pixels *= min_pixels; } } - int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + int max_pixels = conditioner_params.edit_params.cond_refs_max_size; if (max_pixels <= 0) { max_pixels = 384; if (resize_mode == CondResizeMode::AREA) { @@ -2299,14 +2299,14 @@ struct LLMEmbedder : public Conditioner { if (llm->enable_vision && conditioner_params.ref_images != nullptr && !conditioner_params.ref_images->empty()) { std::string img_prompt = ""; const std::string placeholder = "<|image_pad|>"; - int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + int min_pixels = conditioner_params.edit_params.cond_refs_min_size; if (min_pixels <= 0) { min_pixels = 384; if (resize_mode == CondResizeMode::AREA) { min_pixels *= min_pixels; } } - int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + int max_pixels = conditioner_params.edit_params.cond_refs_max_size; if (max_pixels <= 0) { max_pixels = 1024; if (resize_mode == CondResizeMode::AREA) { @@ -2324,6 +2324,7 @@ struct LLMEmbedder : public Conditioner { static_cast(std::round(height / factor)) * static_cast(factor)); int w_bar = std::max(static_cast(factor), static_cast(std::round(width / factor)) * static_cast(factor)); + resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode); LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); @@ -2361,14 +2362,14 @@ struct LLMEmbedder : public Conditioner { min_length = 512 + prompt_template_encode_start_idx; int image_embed_idx = 36 + 6; - int min_pixels = conditioner_params.edit_mode.cond_refs_min_size; + int min_pixels = conditioner_params.edit_params.cond_refs_min_size; if (min_pixels <= 0) { min_pixels = 384; if (resize_mode == CondResizeMode::AREA) { min_pixels *= min_pixels; } } - int max_pixels = conditioner_params.edit_mode.cond_refs_max_size; + int max_pixels = conditioner_params.edit_params.cond_refs_max_size; if (max_pixels <= 0) { max_pixels = 560; if (resize_mode == CondResizeMode::AREA) { diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index 4354aa8fe..d716c7a22 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -22,7 +22,7 @@ struct EditModeParams { Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED; bool force_timestep_0 = false; bool resize_vae_refs = true; - int vae_refs_size = -1; + int vae_refs_max_size = -1; CondResizeMode cond_refs_resize_mode = CondResizeMode::LONGEST_SIDE; int cond_refs_max_size = -1; int cond_refs_min_size = -1; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 7df7f8628..5611d5c34 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2882,6 +2882,7 @@ class StableDiffusionGGML { } } if (preset_name != "default") { + LOG_INFO("Using '%s' preset for edit mode", preset_name.c_str()); params = REF_PRESETS.at(preset_name); } @@ -2912,8 +2913,8 @@ class StableDiffusionGGML { if (!parse_strict_bool(value, params.resize_vae_refs)) { LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); } - } else if (key == "vae_refs_size") { - if (!parse_strict_int(value, params.vae_refs_size)) { + } else if (key == "vae_refs_max_size") { + if (!parse_strict_int(value, params.vae_refs_max_size)) { LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); } } else if (key == "cond_refs_resize_mode") { @@ -2934,12 +2935,27 @@ class StableDiffusionGGML { if (!parse_strict_int(value, params.cond_refs_min_size)) { LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); } - } else if (key != "preset") { + } else if (key != "preset" && key!= "cond_refs_size") { // Optional: catch-all for unknown keys LOG_WARN("ignoring unknown edit mode arg '%s'", key.c_str()); } } + for (const auto& [key, value] : parse_key_value_args(overrides, "edit mode args")) { + if (key == "cond_refs_size") { + int cond_refs_size; + if (!parse_strict_int(value, cond_refs_size)) { + LOG_WARN("ignoring invalid edit mode arg '%s=%s'", key.c_str(), value.c_str()); + } else { + // log something idk + LOG_INFO("cond_refs_size override: setting both min and max size to %ld", (long)cond_refs_size); + params.cond_refs_min_size = cond_refs_size; + params.cond_refs_max_size = cond_refs_size; + } + break; + } + } } + }; /*================================================= SD API ==================================================*/ @@ -4677,7 +4693,7 @@ static std::optional prepare_image_generation_latents(sd sd::Tensor ref_latent; if (edit_params->resize_vae_refs && !sd_version_is_pid(sd_ctx->sd->version)) { LOG_DEBUG("auto resize ref images"); - int target_pixels = edit_params->vae_refs_size > 0 ? edit_params->vae_refs_size : 1024 * 1024; + int target_pixels = edit_params->vae_refs_max_size > 0 ? edit_params->vae_refs_max_size : 1024 * 1024; int vae_image_size = std::min(target_pixels, request->width * request->height); double vae_width = sqrt(vae_image_size * ref_images[i].shape()[0] / ref_images[i].shape()[1]); double vae_height = vae_width * ref_images[i].shape()[1] / ref_images[i].shape()[0]; @@ -4818,6 +4834,8 @@ static std::optional prepare_image_generation_embeds(sd_c condition_params.ref_images = &latents->ref_images; } + condition_params.edit_params = *edit_params; + sd_ctx->sd->prepare_generation_extensions(request->pm_params, request->pulid_params, From 0303dd440a16f58e62b84960eac58ec2840dee91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sat, 11 Jul 2026 21:40:18 +0200 Subject: [PATCH 05/13] use z-image omni preset for boogu image --- src/stable-diffusion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 5611d5c34..a52a7df9f 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2852,7 +2852,7 @@ class StableDiffusionGGML { return "qwen_layered"; } else if (sd_version_is_qwen_image(version)) { return "qwen"; - } else if (sd_version_is_z_image(version)){ + } else if (sd_version_is_z_image(version) || sd_version_is_boogu_image(version)){ return "z_image_omni"; } else if (sd_version_is_krea2(version)){ // have to make a choice between "krea2_edit" mode (for lbouaraba/krea2edit) From 215e0f651b809329fe1d62a7836f8c428cd6e966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sat, 11 Jul 2026 21:46:51 +0200 Subject: [PATCH 06/13] Remove DiffusionParams.ref_index_mode --- src/model/common/rope.hpp | 3 --- src/model/diffusion/flux.hpp | 2 +- src/model/diffusion/model.hpp | 1 - src/model/diffusion/qwen_image.hpp | 2 +- src/model/diffusion/z_image.hpp | 2 +- src/stable-diffusion.cpp | 5 ++--- 6 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/model/common/rope.hpp b/src/model/common/rope.hpp index 04ecde3c5..d2c0269e1 100644 --- a/src/model/common/rope.hpp +++ b/src/model/common/rope.hpp @@ -18,9 +18,6 @@ namespace Rope { DECREASE, }; - __STATIC_INLINE__ RefIndexMode ref_index_mode_from_bool(bool increase_ref_index) { - return increase_ref_index ? RefIndexMode::INCREASE : RefIndexMode::FIXED; - } template __STATIC_INLINE__ std::vector linspace(T start, T end, int num) { diff --git a/src/model/diffusion/flux.hpp b/src/model/diffusion/flux.hpp index e5d795f74..ee3caa22f 100644 --- a/src/model/diffusion/flux.hpp +++ b/src/model/diffusion/flux.hpp @@ -1643,7 +1643,7 @@ namespace Flux { tensor_or_empty(diffusion_params.y), tensor_or_empty(extra->guidance), diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, - diffusion_params.ref_index_mode, + diffusion_params.edit_params.ref_index_mode, extra->skip_layers ? *extra->skip_layers : empty_skip_layers, tensor_or_empty(extra->pulid_id), extra->pulid_id_weight); diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index d716c7a22..83575aa57 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -103,7 +103,6 @@ struct DiffusionParams { const sd::Tensor* y = nullptr; const std::vector>* ref_latents = nullptr; EditModeParams edit_params = {false, false, Rope::RefIndexMode::FIXED, false}; - Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED; DiffusionExtraParams extra = std::monostate{}; }; diff --git a/src/model/diffusion/qwen_image.hpp b/src/model/diffusion/qwen_image.hpp index a7c946fd9..ad47eb4d0 100644 --- a/src/model/diffusion/qwen_image.hpp +++ b/src/model/diffusion/qwen_image.hpp @@ -716,7 +716,7 @@ namespace Qwen { *diffusion_params.timesteps, tensor_or_empty(diffusion_params.context), diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, - diffusion_params.ref_index_mode); + diffusion_params.edit_params.ref_index_mode); } void test() { diff --git a/src/model/diffusion/z_image.hpp b/src/model/diffusion/z_image.hpp index 9fb333cbb..b6ec03160 100644 --- a/src/model/diffusion/z_image.hpp +++ b/src/model/diffusion/z_image.hpp @@ -649,7 +649,7 @@ namespace ZImage { *diffusion_params.timesteps, tensor_or_empty(diffusion_params.context), diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, - diffusion_params.ref_index_mode); + diffusion_params.edit_params.ref_index_mode); } void test() { diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index a52a7df9f..d0f126f05 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2459,9 +2459,8 @@ class StableDiffusionGGML { sd_sample::SampleStepCacheDispatcher step_cache(cache_runtime, step, sigma); std::vector> controls; DiffusionParams diffusion_params; - diffusion_params.x = &noised_input; - diffusion_params.timesteps = ×teps_tensor; - diffusion_params.ref_index_mode = Rope::ref_index_mode_from_bool(increase_ref_index); + diffusion_params.x = &noised_input; + diffusion_params.timesteps = ×teps_tensor; diffusion_params.edit_params = edit_params; sd::guidance::GuidanceInput step_guidance_input; step_guidance_input.step = step; From 14b911cdbba7dabe1d0f4dfd5b30aff5fae96e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sat, 11 Jul 2026 21:55:29 +0200 Subject: [PATCH 07/13] remove increase_ref_index arg from sample() --- src/stable-diffusion.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index d0f126f05..d2532db2b 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2297,7 +2297,6 @@ class StableDiffusionGGML { const char* extra_sample_args, const std::vector& sigmas, const std::vector>& ref_latents, - bool increase_ref_index, EditModeParams& edit_params, const sd::Tensor& denoise_mask, const sd::Tensor& vace_context, @@ -5218,6 +5217,10 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, // TODO: parse EditModeParams from sd_img_gen_params->ref_image_mode sd_ctx->sd->set_edit_mode_params(edit_params, sd_img_gen_params->ref_image_mode); + if (request.increase_ref_index){ + edit_params.ref_index_mode = Rope::RefIndexMode::INCREASE; + } + ImageVaeAxesGuard axes_guard(sd_ctx, sd_img_gen_params, request); SamplePlan plan(sd_ctx, sd_img_gen_params, request); @@ -5282,7 +5285,6 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, plan.extra_sample_args, plan.sigmas, latents.ref_latents, - request.increase_ref_index, edit_params, latents.denoise_mask, sd::Tensor(), @@ -5404,7 +5406,6 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, plan.extra_sample_args, hires_sigma_sched, latents.ref_latents, - request.increase_ref_index, edit_params, hires_denoise_mask, sd::Tensor(), @@ -6167,7 +6168,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, plan.high_noise_extra_sample_args, high_noise_sigmas, std::vector>{}, - false, edit_params, latents.denoise_mask, latents.vace_context, @@ -6210,7 +6210,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, plan.extra_sample_args, plan.sigmas, std::vector>{}, - false, edit_params, latents.denoise_mask, latents.vace_context, @@ -6349,7 +6348,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, plan.extra_sample_args, hires_sigma_sched, std::vector>{}, - false, edit_params, hires_denoise_mask, sd::Tensor(), From 4f546164cfd11528ab1a0064e5f0cb695db2b2be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sun, 12 Jul 2026 00:32:25 +0200 Subject: [PATCH 08/13] misc --- src/model/diffusion/boogu.hpp | 2 +- src/model/diffusion/flux.hpp | 2 +- src/model/diffusion/qwen_image.hpp | 2 +- src/model/diffusion/z_image.hpp | 2 +- src/stable-diffusion.cpp | 3 +++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/model/diffusion/boogu.hpp b/src/model/diffusion/boogu.hpp index 9ab2dccbc..66323159d 100644 --- a/src/model/diffusion/boogu.hpp +++ b/src/model/diffusion/boogu.hpp @@ -827,7 +827,7 @@ namespace Boogu { *diffusion_params.x, *diffusion_params.timesteps, tensor_or_empty(diffusion_params.context), - diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents); + diffusion_params.ref_latents && diffusion_params.edit_params.use_dit_refs ? *diffusion_params.ref_latents : empty_ref_latents); } }; } // namespace Boogu diff --git a/src/model/diffusion/flux.hpp b/src/model/diffusion/flux.hpp index ee3caa22f..643120157 100644 --- a/src/model/diffusion/flux.hpp +++ b/src/model/diffusion/flux.hpp @@ -1642,7 +1642,7 @@ namespace Flux { tensor_or_empty(diffusion_params.c_concat), tensor_or_empty(diffusion_params.y), tensor_or_empty(extra->guidance), - diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, + diffusion_params.ref_latents && diffusion_params.edit_params.use_dit_refs ? *diffusion_params.ref_latents : empty_ref_latents, diffusion_params.edit_params.ref_index_mode, extra->skip_layers ? *extra->skip_layers : empty_skip_layers, tensor_or_empty(extra->pulid_id), diff --git a/src/model/diffusion/qwen_image.hpp b/src/model/diffusion/qwen_image.hpp index ad47eb4d0..37fc26ab0 100644 --- a/src/model/diffusion/qwen_image.hpp +++ b/src/model/diffusion/qwen_image.hpp @@ -715,7 +715,7 @@ namespace Qwen { *diffusion_params.x, *diffusion_params.timesteps, tensor_or_empty(diffusion_params.context), - diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, + diffusion_params.ref_latents && diffusion_params.edit_params.use_dit_refs ? *diffusion_params.ref_latents : empty_ref_latents, diffusion_params.edit_params.ref_index_mode); } diff --git a/src/model/diffusion/z_image.hpp b/src/model/diffusion/z_image.hpp index b6ec03160..90f4064d3 100644 --- a/src/model/diffusion/z_image.hpp +++ b/src/model/diffusion/z_image.hpp @@ -648,7 +648,7 @@ namespace ZImage { *diffusion_params.x, *diffusion_params.timesteps, tensor_or_empty(diffusion_params.context), - diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, + diffusion_params.ref_latents && diffusion_params.edit_params.use_dit_refs ? *diffusion_params.ref_latents : empty_ref_latents, diffusion_params.edit_params.ref_index_mode); } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index d2532db2b..ddc52f2ad 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2952,6 +2952,9 @@ class StableDiffusionGGML { break; } } + if (params.force_timestep_0 && !sd_version_is_krea2(version)){ + LOG_WARN("force_timestep_0 is only supported by Krea2 architecture for now"); + } } }; From 3c07bf943fa861134f96361cfdc2aef4407cc1d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sun, 12 Jul 2026 11:40:46 +0200 Subject: [PATCH 09/13] update docs --- README.md | 2 +- docs/edit.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 docs/edit.md diff --git a/README.md b/README.md index bf722882c..8daa162f4 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ API and command-line option may change frequently.*** - [SeFi-Image](./docs/sefi_image.md) - [HiDream-O1-Image](./docs/hidream_o1_image.md) - [Ideogram4](./docs/ideogram4.md) - - Image Edit Models + - [Image Edit Models](./docs/edit.md) - [FLUX.1-Kontext-dev](./docs/kontext.md) - [Qwen Image Edit series](./docs/qwen_image_edit.md) - [LongCat Image Edit](./docs/longcat_image.md) diff --git a/docs/edit.md b/docs/edit.md new file mode 100644 index 000000000..cde421593 --- /dev/null +++ b/docs/edit.md @@ -0,0 +1,67 @@ +# Image Editing + +Image editing in `stable-diffusion.cpp` allows you to use reference images to guide the generation process, enabling tasks like identity preservation, style transfer, or layout modification. + +## Supported Models + +Depending on the architecture, different models handle reference images differently. + +| Model | Default Preset | +| :--- | :--- | +| [**FLUX.1-Kontext-dev**](./kontext.md) | `flux_kontext` | +| [**LongCat Image Edit**](./longcat_image.md) | `flux_kontext` | +| [**Qwen Image Edit**](./qwen_image_edit.md) | `qwen` | +| **Qwen Image LAYERED** | `qwen_layered` | +| [**Flux.2 [Dev] / Flux.2 [Klein]**](./flux2.md) | `flux2` | +| [**Boogu Image Edit**](./boogu_image.md) | `z_image_omni` | +| **Krea2 (Community Edit LoRAs)** | `krea2_ostris_edit` | + +--- + +## Configuring Reference Modes (`--ref-image-mode`) + +Different DiT-based editing models require different configurations to process reference images correctly (e.g., whether to use a Vision Language Model (VLM) encoder or pass VAE-encoded images directly to the DiT). + +To simplify this, we provide **Presets**. By default, the system automatically selects the best preset based on the model architecture. However, you can override this using the `--ref-image-mode` argument. + +### Usage +The `--ref-image-mode` argument accepts a comma-separated list of key-value pairs: + +**Using a preset:** +`--ref-image-mode "preset=qwen_layered"` + +**Using a preset with a specific override:** +`--ref-image-mode "preset=krea2_edit,force_timestep_0=true"` + +### Available Presets + +| Preset | Primary Use Case | +| :--- | :--- | +| `flux_kontext` | FLUX.1 Kontext, LongCat Image Edit | +| `flux2` | FLUX.2 models | +| `qwen` | Qwen Image Edit | +| `qwen_layered` | Qwen Image Layered | +| `z_image_omni` | Boogu, Z-Image Omni | +| `krea2_ostris_edit` | Most Krea2 Community edit LoRAs (trained with Ostris script) | +| `krea2_edit` | Specifically for [lbouaraba/krea2edit](https://huggingface.co/conradlocke/krea2-identity-edit). (or similar) | +| `default` | Uses the automatic detection based on model architecture. | + +--- + +## Advanced Parameter Reference + +If presets are insufficient, you can manually configure the following parameters via `--ref-image-mode`: + +| Key | Type | Description | +| :--- | :--- | :--- | +| `preset` | string | Sets a predefined group of parameters. | +| `use_vlm` | bool | Whether references are passed to the VLM encoder (if the model supports it). | +| `pass_to_dit` | bool | Whether VAE-encoded references are passed directly to the DiT. | +| `ref_index_mode` | enum | Behavior of the RoPE index: `fixed`, `increase`, or `decrease`. | +| `force_timestep_0` | bool | Forces timestep=0 for reference tokens (Krea2 architecture only). | +| `resize_vae_refs` | bool | Whether to resize VAE references. | +| `vae_refs_max_size` | int | Maximum pixel size for VAE references. | +| `cond_refs_resize_mode` | enum | How to resize condition references: `longest_side`, `area`, or `none`. | +| `cond_refs_max_size` | int | Maximum pixel size for condition references. | +| `cond_refs_min_size` | int | Minimum pixel size for condition references. | +| `cond_refs_size` | int | Shortcut to set both min and max size to the same value. | From 9e78f0de3d26a2fed7ca994d18c42c83e5a2c80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sun, 12 Jul 2026 11:59:49 +0200 Subject: [PATCH 10/13] API: deprecate increase_ref_index and auto_resize_ref_image --- examples/common/common.cpp | 18 ++++++++++++++++-- include/stable-diffusion.h | 2 -- src/stable-diffusion.cpp | 14 ++------------ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index e9f7f85f1..82c614b72 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -2423,6 +2423,22 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { pulid_id_weight, }; + if (!auto_resize_ref_image) { + if (!ref_mode.empty()) { + ref_mode += ","; + } + ref_mode += "resize_vae_refs=0"; + LOG_WARN("Notice: --disable-auto-resize-ref-image is deprecated. Use --ref-image-mode \"resize_vae_refs=off\" instead."); + } + + if (increase_ref_index) { + if (!ref_mode.empty()) { + ref_mode += ","; + } + ref_mode += "ref_index_mode=increase"; + LOG_WARN("Notice: --increase-ref-index is deprecated. Use --ref-image-mode \"ref_index_mode=increase\" instead."); + } + params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); params.lora_count = static_cast(lora_vec.size()); params.prompt = prompt.c_str(); @@ -2431,8 +2447,6 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { params.init_image = init_image.get(); params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); params.ref_images_count = static_cast(ref_image_views.size()); - params.auto_resize_ref_image = auto_resize_ref_image; - params.increase_ref_index = increase_ref_index; params.ref_image_mode = ref_mode.c_str(); params.mask_image = mask_image.get(); params.width = get_resolved_width(); diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 3bfd92a49..8c12f299e 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -363,8 +363,6 @@ typedef struct { sd_image_t init_image; sd_image_t* ref_images; int ref_images_count; - bool auto_resize_ref_image; - bool increase_ref_index; const char* ref_image_mode; sd_image_t mask_image; int width; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index ddc52f2ad..4c36affce 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -3455,8 +3455,7 @@ char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params) { "batch_count: %d\n" "qwen_image_layers: %d\n" "ref_images_count: %d\n" - "auto_resize_ref_image: %s\n" - "increase_ref_index: %s\n" + "ref_image_mode: %s\n" "control_strength: %.2f\n" "photo maker: {style_strength = %.2f, id_images_count = %d, id_embed_path = %s}\n" "VAE tiling: %s (temporal=%s, extra_tiling_args=%s)\n" @@ -3474,8 +3473,7 @@ char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params) { sd_img_gen_params->batch_count, sd_img_gen_params->qwen_image_layers, sd_img_gen_params->ref_images_count, - BOOL_STR(sd_img_gen_params->auto_resize_ref_image), - BOOL_STR(sd_img_gen_params->increase_ref_index), + sd_img_gen_params->ref_image_mode, sd_img_gen_params->control_strength, sd_img_gen_params->pm_params.style_strength, sd_img_gen_params->pm_params.id_images_count, @@ -3772,8 +3770,6 @@ struct GenerationRequest { float strength = 1.f; float control_strength = 0.f; float eta = 0.f; - bool increase_ref_index = false; - bool auto_resize_ref_image = false; sd_guidance_params_t guidance = {}; sd_guidance_params_t high_noise_guidance = {}; sd_pm_params_t pm_params = {}; @@ -3799,8 +3795,6 @@ struct GenerationRequest { strength = sd_img_gen_params->strength; control_strength = sd_img_gen_params->control_strength; eta = sd_img_gen_params->sample_params.eta; - increase_ref_index = sd_img_gen_params->increase_ref_index; - auto_resize_ref_image = sd_img_gen_params->auto_resize_ref_image; has_ref_images = sd_img_gen_params->ref_images_count > 0; guidance = sd_img_gen_params->sample_params.guidance; pm_params = sd_img_gen_params->pm_params; @@ -5220,10 +5214,6 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, // TODO: parse EditModeParams from sd_img_gen_params->ref_image_mode sd_ctx->sd->set_edit_mode_params(edit_params, sd_img_gen_params->ref_image_mode); - if (request.increase_ref_index){ - edit_params.ref_index_mode = Rope::RefIndexMode::INCREASE; - } - ImageVaeAxesGuard axes_guard(sd_ctx, sd_img_gen_params, request); SamplePlan plan(sd_ctx, sd_img_gen_params, request); From 4725e3d132683e54e858a85e4074292b46ac0f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Sun, 12 Jul 2026 12:21:45 +0200 Subject: [PATCH 11/13] docs: improve edit.md --- docs/edit.md | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/docs/edit.md b/docs/edit.md index cde421593..df768111f 100644 --- a/docs/edit.md +++ b/docs/edit.md @@ -52,16 +52,34 @@ The `--ref-image-mode` argument accepts a comma-separated list of key-value pair If presets are insufficient, you can manually configure the following parameters via `--ref-image-mode`: -| Key | Type | Description | -| :--- | :--- | :--- | -| `preset` | string | Sets a predefined group of parameters. | -| `use_vlm` | bool | Whether references are passed to the VLM encoder (if the model supports it). | -| `pass_to_dit` | bool | Whether VAE-encoded references are passed directly to the DiT. | -| `ref_index_mode` | enum | Behavior of the RoPE index: `fixed`, `increase`, or `decrease`. | -| `force_timestep_0` | bool | Forces timestep=0 for reference tokens (Krea2 architecture only). | -| `resize_vae_refs` | bool | Whether to resize VAE references. | -| `vae_refs_max_size` | int | Maximum pixel size for VAE references. | -| `cond_refs_resize_mode` | enum | How to resize condition references: `longest_side`, `area`, or `none`. | -| `cond_refs_max_size` | int | Maximum pixel size for condition references. | -| `cond_refs_min_size` | int | Minimum pixel size for condition references. | -| `cond_refs_size` | int | Shortcut to set both min and max size to the same value. | +| Key | Type | Description | Allowed Values | +| :--- | :--- | :--- | :--- | +| `preset` | string | Overrides the automatic preset. | (See the Presets table above) | +| `use_vlm` | bool | Whether references are passed to the VLM encoder. | `true`, `false` | +| `pass_to_dit` | bool | Whether VAE-encoded references are passed directly to the DiT. | `true`, `false` | +| `ref_index_mode` | string | Behavior of the RoPE index. | `fixed`, `increase`, `decrease` | +| `force_timestep_0` | bool | Forces timestep=0 for reference tokens. | `true`, `false` (Krea2 only) | +| `resize_vae_refs` | bool | Whether to resize VAE references. | `true`, `false` | +| `vae_refs_max_size` | int | Maximum pixel size for VAE references. | Integer | +| `cond_refs_resize_mode` | string | How to resize condition references. | `longest_side`, `area`, `none` | +| `cond_refs_max_size` | int | Maximum pixel size for condition references. | Integer | +| `cond_refs_min_size` | int | Minimum pixel size for condition references. | Integer | +| `cond_refs_size` | int | Shortcut to set both min and max size to the same value. | Integer | + +### Preset Default Values + +For a technical overview of how each preset is configured, see the table below. + +| Preset | VLM | RoPE Index | Cond Resize | Special Notes | +| :--- | :---: | :---: | :---: | :--- | +| `flux_kontext` | No | `fixed` | `none` | | +| `flux2` | No | `increase` | `none` | | +| `qwen` | Yes | `increase` | `area` | | +| `qwen_layered` | Yes | `decrease` | `area` | | +| `z_image_omni` | Yes | `fixed` | `area` | | +| `krea2_ostris_edit`| Yes | `increase` | `area` | `force_timestep_0 = true` | +| `krea2_edit` | Yes | `increase` | `longest` | `cond_refs_size = 768` | + +**Additional Default Notes:** +- **Condition Sizes:** For most presets, `cond_refs_max_size` and `cond_refs_min_size` are set to `-1`, meaning the values are model-dependent and handled automatically. +- **VAE Reference Size:** `vae_refs_max_size` defaults to $1024 \times 1024$ pixels (`1048576`). From 0a0fdc5884559ee7a374c2d1d95b1de2dfad7d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Mon, 13 Jul 2026 17:51:58 +0200 Subject: [PATCH 12/13] add anima edit ("cosmos_reference") support --- src/model/diffusion/anima.hpp | 61 ++++++++++++++++++++++++----------- src/model/diffusion/model.hpp | 1 + src/stable-diffusion.cpp | 8 +++-- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/src/model/diffusion/anima.hpp b/src/model/diffusion/anima.hpp index 1c1ec2ed2..4476eb800 100644 --- a/src/model/diffusion/anima.hpp +++ b/src/model/diffusion/anima.hpp @@ -484,10 +484,11 @@ namespace Anima { ggml_tensor* timestep, ggml_tensor* encoder_hidden_states, ggml_tensor* image_pe, - ggml_tensor* t5_ids = nullptr, - ggml_tensor* t5_weights = nullptr, - ggml_tensor* adapter_q_pe = nullptr, - ggml_tensor* adapter_k_pe = nullptr) { + ggml_tensor* t5_ids = nullptr, + ggml_tensor* t5_weights = nullptr, + ggml_tensor* adapter_q_pe = nullptr, + ggml_tensor* adapter_k_pe = nullptr, + std::vector ref_latents = {}) { GGML_ASSERT(x->ne[3] == 1); auto x_embedder = std::dynamic_pointer_cast(blocks["x_embedder"]); @@ -502,8 +503,16 @@ namespace Anima { auto padding_mask = ggml_ext_zeros(ctx->ggml_ctx, x->ne[0], x->ne[1], 1, x->ne[3]); x = ggml_concat(ctx->ggml_ctx, x, padding_mask, 2); // [N, C + 1, H, W] - x = DiT::pad_and_patchify(ctx, x, config.patch_size, config.patch_size); // [N, h*w, (C+1)*ph*pw] - + x = DiT::pad_and_patchify(ctx, x, config.patch_size, config.patch_size); // [N, h*w, (C+1)*ph*pw] + int64_t img_len = x->ne[1]; + if (ref_latents.size() > 0) { + for (ggml_tensor* ref : ref_latents) { + auto padding_mask = ggml_ext_zeros(ctx->ggml_ctx, ref->ne[0], ref->ne[1], 1, ref->ne[3]); + ref = ggml_concat(ctx->ggml_ctx, ref, padding_mask, 2); // [N, C + 1, H, W] + ref = DiT::pad_and_patchify(ctx, ref, config.patch_size, config.patch_size); + x = ggml_concat(ctx->ggml_ctx, x, ref, 1); + } + } x = x_embedder->forward(ctx, x); auto timestep_proj = ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep, static_cast(config.hidden_size)); @@ -543,6 +552,7 @@ namespace Anima { x = block->forward(ctx, x, encoder_hidden_states, embedded_timestep, temb, image_pe); sd::ggml_graph_cut::mark_graph_cut(x, "anima.blocks." + std::to_string(i), "x"); } + x = ggml_ext_slice(ctx->ggml_ctx, x, 1, 0, img_len); x = final_layer->forward(ctx, x, embedded_timestep, temb); // [N, h*w, ph*pw*C] @@ -602,8 +612,8 @@ namespace Anima { const std::vector& axes_dim, float h_extrapolation_ratio, float w_extrapolation_ratio, - float t_extrapolation_ratio) { - static const std::vector empty_ref_latents; + float t_extrapolation_ratio, + const std::vector& ref_latents) { auto ids = Rope::gen_flux_ids(h, w, patch_size, @@ -611,7 +621,7 @@ namespace Anima { static_cast(axes_dim.size()), 0, {}, - empty_ref_latents, + ref_latents, Rope::RefIndexMode::FIXED, 1.0f, false); @@ -626,14 +636,20 @@ namespace Anima { ggml_cgraph* build_graph(const sd::Tensor& x_tensor, const sd::Tensor& timesteps_tensor, - const sd::Tensor& context_tensor = {}, - const sd::Tensor& t5_ids_tensor = {}, - const sd::Tensor& t5_weights_tensor = {}) { + const sd::Tensor& context_tensor = {}, + const sd::Tensor& t5_ids_tensor = {}, + const sd::Tensor& t5_weights_tensor = {}, + const std::vector>& ref_latents_tensor = {}) { ggml_tensor* x = make_input(x_tensor); ggml_tensor* timesteps = make_input(timesteps_tensor); ggml_tensor* context = make_optional_input(context_tensor); ggml_tensor* t5_ids = make_optional_input(t5_ids_tensor); ggml_tensor* t5_weights = make_optional_input(t5_weights_tensor); + std::vector ref_latents; + ref_latents.reserve(ref_latents_tensor.size()); + for (const auto& ref_latent_tensor : ref_latents_tensor) { + ref_latents.push_back(make_input(ref_latent_tensor)); + } GGML_ASSERT(x->ne[3] == 1); ggml_cgraph* gf = new_graph_custom(ANIMA_GRAPH_SIZE); @@ -650,7 +666,8 @@ namespace Anima { config.axes_dim, 4.0f, 4.0f, - 1.0f); + 1.0f, + ref_latents); int64_t image_pos_len = static_cast(image_pe_vec.size()) / (2 * 2 * (config.head_dim / 2)); auto image_pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, image_pos_len); set_backend_tensor_data(image_pe, image_pe_vec.data()); @@ -682,7 +699,8 @@ namespace Anima { t5_ids, t5_weights, adapter_q_pe, - adapter_k_pe); + adapter_k_pe, + ref_latents); ggml_build_forward_expand(gf, out); return gf; @@ -691,11 +709,13 @@ namespace Anima { sd::Tensor compute(int n_threads, const sd::Tensor& x, const sd::Tensor& timesteps, - const sd::Tensor& context = {}, - const sd::Tensor& t5_ids = {}, - const sd::Tensor& t5_weights = {}) { + const sd::Tensor& context = {}, + const sd::Tensor& t5_ids = {}, + const sd::Tensor& t5_weights = {}, + const std::vector>& ref_latents = {}, + EditModeParams edit_params = REF_PRESETS.at("cosmos_reference")) { auto get_graph = [&]() -> ggml_cgraph* { - return build_graph(x, timesteps, context, t5_ids, t5_weights); + return build_graph(x, timesteps, context, t5_ids, t5_weights, ref_latents); }; return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } @@ -705,12 +725,15 @@ namespace Anima { GGML_ASSERT(diffusion_params.x != nullptr); GGML_ASSERT(diffusion_params.timesteps != nullptr); const auto* extra = diffusion_extra_as(diffusion_params); + static const std::vector> empty_ref_latents; return compute(n_threads, *diffusion_params.x, *diffusion_params.timesteps, tensor_or_empty(diffusion_params.context), tensor_or_empty(extra->t5_ids), - tensor_or_empty(extra->t5_weights)); + tensor_or_empty(extra->t5_weights), + diffusion_params.ref_latents && diffusion_params.edit_params.use_dit_refs ? *diffusion_params.ref_latents : empty_ref_latents, + diffusion_params.edit_params); } }; } // namespace Anima diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index 83575aa57..354442fc2 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -36,6 +36,7 @@ const std::unordered_map REF_PRESETS = { {"z_image_omni", {true, true, Rope::RefIndexMode::FIXED, false, true, -1, CondResizeMode::AREA, -1, -1}}, {"krea2_ostris_edit", {true, true, Rope::RefIndexMode::INCREASE, true, true, -1, CondResizeMode::AREA, -1, -1}}, {"krea2_edit", {true, true, Rope::RefIndexMode::INCREASE, false, true, -1, CondResizeMode::LONGEST_SIDE, 768, 768}}, + {"cosmos_reference", {false, true, Rope::RefIndexMode::INCREASE, false, false, -1, CondResizeMode::NONE, -1, -1}}, }; struct UNetDiffusionExtra { diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 4c36affce..b0a188e44 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2846,17 +2846,19 @@ class StableDiffusionGGML { return "flux_kontext"; } else if (sd_version_is_flux2(version) || sd_version_is_sefi_image(version)) { return "flux2"; - } else if (version == VERSION_QWEN_IMAGE_LAYERED){ + } else if (version == VERSION_QWEN_IMAGE_LAYERED) { return "qwen_layered"; } else if (sd_version_is_qwen_image(version)) { return "qwen"; - } else if (sd_version_is_z_image(version) || sd_version_is_boogu_image(version)){ + } else if (sd_version_is_z_image(version) || sd_version_is_boogu_image(version)) { return "z_image_omni"; - } else if (sd_version_is_krea2(version)){ + } else if (sd_version_is_krea2(version)) { // have to make a choice between "krea2_edit" mode (for lbouaraba/krea2edit) // and "krea2_ostris_edit" (for krea2 ostris edit) // since krea2 ostris edit support predates, it should probably be default return "krea2_ostris_edit"; + } else if (sd_version_is_anima(version)) { + return "cosmos_reference"; } return "default"; } From 5c8b366579af888c33366e2043065aa278b570ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Mon, 13 Jul 2026 18:13:55 +0200 Subject: [PATCH 13/13] docs: update docs --- docs/edit.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/edit.md b/docs/edit.md index df768111f..c28654e20 100644 --- a/docs/edit.md +++ b/docs/edit.md @@ -1,6 +1,7 @@ # Image Editing Image editing in `stable-diffusion.cpp` allows you to use reference images to guide the generation process, enabling tasks like identity preservation, style transfer, or layout modification. + ## Supported Models @@ -15,6 +16,9 @@ Depending on the architecture, different models handle reference images differen | [**Flux.2 [Dev] / Flux.2 [Klein]**](./flux2.md) | `flux2` | | [**Boogu Image Edit**](./boogu_image.md) | `z_image_omni` | | **Krea2 (Community Edit LoRAs)** | `krea2_ostris_edit` | +| **Anima (Community Edit LoRAs)** | `cosmos_reference` | + +Stable-diffusion.spp also supports basic Unet-based editing models like instruct-pix2pix or CosXL-Edit. This document is not about those. --- @@ -44,6 +48,7 @@ The `--ref-image-mode` argument accepts a comma-separated list of key-value pair | `z_image_omni` | Boogu, Z-Image Omni | | `krea2_ostris_edit` | Most Krea2 Community edit LoRAs (trained with Ostris script) | | `krea2_edit` | Specifically for [lbouaraba/krea2edit](https://huggingface.co/conradlocke/krea2-identity-edit). (or similar) | +| `cosmos_reference` | For Anima | | `default` | Uses the automatic detection based on model architecture. | --- @@ -79,6 +84,7 @@ For a technical overview of how each preset is configured, see the table below. | `z_image_omni` | Yes | `fixed` | `area` | | | `krea2_ostris_edit`| Yes | `increase` | `area` | `force_timestep_0 = true` | | `krea2_edit` | Yes | `increase` | `longest` | `cond_refs_size = 768` | +| `cosmos_reference` | No | `fixed` | `none` | `resize_vae_refs = false` | **Additional Default Notes:** - **Condition Sizes:** For most presets, `cond_refs_max_size` and `cond_refs_min_size` are set to `-1`, meaning the values are model-dependent and handled automatically.