From 781b396e8acae97e9e6bfb955611e4a51d4a43d0 Mon Sep 17 00:00:00 2001 From: aegioscy Date: Mon, 27 Jul 2026 10:06:51 +0200 Subject: [PATCH 1/4] feat(ltx): add Ingredients reference conditioning Expose static reference-sheet conditioning through the LTX video API and CLI so IC-LoRA adapters can preserve reference content during generation. --- examples/common/common.cpp | 33 +++++++++++- examples/common/common.h | 2 + include/stable-diffusion.h | 11 ++++ src/stable-diffusion.cpp | 102 ++++++++++++++++++++++++++++++++++++- 4 files changed, 146 insertions(+), 2 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 9a22efe6a..88625c853 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1027,6 +1027,14 @@ ArgOptions SDGenerationParams::get_options() { "--vace-strength", "wan vace strength", &vace_strength}, + {"", + "--reference-attention-strength", + "LTX IC-LoRA reference conditioning strength in [0, 1] (default: 1.0)", + &reference_attention_strength}, + {"", + "--reference-downscale-factor", + "LTX IC-LoRA reference image downscale factor (currently must be 1.0; default: 1.0)", + &reference_downscale_factor}, {"", "--vae-tile-overlap", "tile overlap for vae tiling, in fraction of tile size (default: 0.5)", @@ -1366,7 +1374,7 @@ ArgOptions SDGenerationParams::get_options() { on_high_noise_skip_layers_arg}, {"-r", "--ref-image", - "reference image for Flux Kontext models (can be used multiple times)", + "reference image for Flux Kontext or LTX IC-LoRA video models (can be used multiple times)", on_ref_image_arg}, {"", "--cache-mode", @@ -2141,6 +2149,17 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } + if (mode == VID_GEN && !ref_image_paths.empty()) { + if (reference_attention_strength < 0.f || reference_attention_strength > 1.f) { + LOG_ERROR("error: reference attention strength must be in [0, 1]"); + return false; + } + if (reference_downscale_factor != 1.f) { + LOG_ERROR("error: LTX IC-LoRA currently requires reference downscale factor to be 1"); + return false; + } + } + if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) { LOG_ERROR("error: shifted_timestep must be in range [0, 1000]"); return false; @@ -2303,6 +2322,12 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { control_frame_views.push_back(frame.get()); } + ref_image_views.clear(); + ref_image_views.reserve(ref_images.size()); + for (auto& ref_image : ref_images) { + ref_image_views.push_back(ref_image.get()); + } + sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data(); sample_params.guidance.slg.layer_count = skip_layers.size(); high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data(); @@ -2323,6 +2348,10 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { params.end_image = end_image.get(); params.control_frames = control_frame_views.empty() ? nullptr : control_frame_views.data(); params.control_frames_size = static_cast(control_frame_views.size()); + params.reference_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); + params.reference_images_count = static_cast(ref_image_views.size()); + params.reference_attention_strength = reference_attention_strength; + params.reference_downscale_factor = reference_downscale_factor; params.width = get_resolved_width(); params.height = get_resolved_height(); params.sample_params = sample_params; @@ -2415,6 +2444,8 @@ std::string SDGenerationParams::to_string() const { << " video_frames: " << video_frames << ",\n" << " fps: " << fps << ",\n" << " vace_strength: " << vace_strength << ",\n" + << " reference_attention_strength: " << reference_attention_strength << ",\n" + << " reference_downscale_factor: " << reference_downscale_factor << ",\n" << " strength: " << strength << ",\n" << " control_strength: " << control_strength << ",\n" << " seed: " << seed << ",\n" diff --git a/examples/common/common.h b/examples/common/common.h index 999d23d52..41af8aad0 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -192,6 +192,8 @@ struct SDGenerationParams { int video_frames = 1; int fps = 16; float vace_strength = 1.f; + float reference_attention_strength = 1.f; + float reference_downscale_factor = 1.f; sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; std::string extra_tiling_args; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index f2b6f00fc..c62dfc846 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -404,6 +404,17 @@ typedef struct { sd_tiling_params_t vae_tiling_params; sd_cache_params_t cache; sd_hires_params_t hires; + // LTX IC-LoRA reference conditioning. These fields are ignored for + // non-LTX video models. Each input is a reference sheet/image that is + // expanded into a static reference video before VAE encoding. + sd_image_t* reference_images; + int reference_images_count; + // [0, 1]. Zero disables the reference tokens; one preserves their full + // conditioning weight. Defaults are set by sd_vid_gen_params_init(). + float reference_attention_strength; + // The current LTX Ingredients workflow requires 1.0. Values other than + // one are rejected until a spatial downscale implementation is added. + float reference_downscale_factor; } sd_vid_gen_params_t; typedef struct sd_ctx_t sd_ctx_t; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index df84281b2..7fad365d0 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -3082,6 +3082,10 @@ void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) { sd_vid_gen_params->fps = 16; sd_vid_gen_params->moe_boundary = 0.875f; sd_vid_gen_params->vace_strength = 1.f; + sd_vid_gen_params->reference_images = nullptr; + sd_vid_gen_params->reference_images_count = 0; + sd_vid_gen_params->reference_attention_strength = 1.f; + sd_vid_gen_params->reference_downscale_factor = 1.f; sd_vid_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; sd_vid_gen_params->hires.enabled = false; sd_vid_gen_params->hires.upscaler = SD_HIRES_UPSCALER_LATENT; @@ -3827,6 +3831,30 @@ static sd::Tensor make_ltxav_video_denoise_mask(const sd::Tensor& value); } +static sd::Tensor make_ltxav_static_reference_video(const sd::Tensor& image, int frames) { + if (image.empty() || image.dim() != 4 || frames <= 0) { + return {}; + } + + const int64_t width = image.shape()[0]; + const int64_t height = image.shape()[1]; + const int64_t channels = image.shape()[2]; + const int64_t batches = image.shape()[3]; + const int64_t plane = width * height; + sd::Tensor video({width, height, frames, channels, batches}); + + for (int64_t batch = 0; batch < batches; ++batch) { + for (int64_t channel = 0; channel < channels; ++channel) { + const float* source = image.data() + (batch * channels + channel) * plane; + for (int frame = 0; frame < frames; ++frame) { + float* destination = video.data() + ((batch * channels + channel) * frames + frame) * plane; + std::copy_n(source, plane, destination); + } + } + } + return video; +} + static sd::Tensor encode_ltxav_condition_image(sd_ctx_t* sd_ctx, const sd::Tensor& image, const char* name) { @@ -4812,6 +4840,7 @@ static std::optional prepare_video_generation_latents(sd sd::Tensor start_image; sd::Tensor end_image; + std::vector> reference_images; if (sd_vid_gen_params->init_image.data) { start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height); @@ -4820,10 +4849,30 @@ static std::optional prepare_video_generation_latents(sd if (sd_vid_gen_params->end_image.data) { end_image = sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height); } + if (sd_vid_gen_params->reference_images_count < 0 || + (sd_vid_gen_params->reference_images_count > 0 && sd_vid_gen_params->reference_images == nullptr)) { + LOG_ERROR("invalid LTX IC-LoRA reference image inputs"); + return std::nullopt; + } + for (int i = 0; i < sd_vid_gen_params->reference_images_count; ++i) { + const sd_image_t& image = sd_vid_gen_params->reference_images[i]; + if (image.data == nullptr) { + LOG_ERROR("LTX IC-LoRA reference image %d is empty", i); + return std::nullopt; + } + reference_images.push_back(sd_image_to_tensor(image, request->width, request->height)); + if (reference_images.back().empty()) { + LOG_ERROR("failed to prepare LTX IC-LoRA reference image %d", i); + return std::nullopt; + } + } if (sd_version_is_ltxav(sd_ctx->sd->version)) { latents.audio_length = get_ltxav_num_audio_latents(request->frames, request->fps); latents.audio_latent = make_ltxav_empty_audio_latent(latents.audio_length); + } else if (!reference_images.empty()) { + LOG_ERROR("IC-LoRA reference conditioning is supported only for LTX video models"); + return std::nullopt; } if (sd_version_is_ltxav(sd_ctx->sd->version)) { @@ -4832,7 +4881,24 @@ static std::optional prepare_video_generation_latents(sd return std::nullopt; } - if (!start_image.empty() || !end_image.empty()) { + if (!reference_images.empty() && + (!start_image.empty() || !end_image.empty())) { + LOG_ERROR("LTX IC-LoRA reference conditioning cannot be combined with init_image or end_image"); + return std::nullopt; + } + if (!reference_images.empty()) { + if (sd_vid_gen_params->reference_downscale_factor != 1.f) { + LOG_ERROR("LTX IC-LoRA currently requires reference_downscale_factor=1"); + return std::nullopt; + } + if (sd_vid_gen_params->reference_attention_strength < 0.f || + sd_vid_gen_params->reference_attention_strength > 1.f) { + LOG_ERROR("LTX IC-LoRA reference_attention_strength must be in [0, 1]"); + return std::nullopt; + } + } + + if (!start_image.empty() || !end_image.empty() || !reference_images.empty()) { if (sd_ctx->sd->vae_decode_only) { LOG_ERROR("LTXAV image conditioning requires VAE encoder weights; create the context with vae_decode_only=false"); return std::nullopt; @@ -4842,6 +4908,8 @@ static std::optional prepare_video_generation_latents(sd LOG_INFO("FLF2V"); } else if (!start_image.empty()) { LOG_INFO("IMG2VID"); + } else if (!reference_images.empty()) { + LOG_INFO("LTX IC-LoRA reference conditioning"); } else { LOG_INFO("END2VID"); } @@ -4919,6 +4987,38 @@ static std::optional prepare_video_generation_latents(sd } } + if (!reference_images.empty()) { + sd::Tensor reference_latents; + for (size_t i = 0; i < reference_images.size(); ++i) { + auto static_reference = + make_ltxav_static_reference_video(reference_images[i], request->frames); + auto reference_latent = sd_ctx->sd->encode_first_stage(static_reference); + if (reference_latent.empty()) { + LOG_ERROR("failed to encode LTX IC-LoRA reference image %zu", i); + return std::nullopt; + } + reference_latents = reference_latents.empty() + ? std::move(reference_latent) + : sd::ops::concat(reference_latents, reference_latent, 2); + } + if (!apply_video_condition_by_keyframe_index(reference_latents, + 0, + "IC-LoRA reference")) { + return std::nullopt; + } + const float reference_strength = + std::clamp(sd_vid_gen_params->reference_attention_strength, 0.f, 1.f); + const int64_t reference_frames = reference_latents.shape()[2]; + sd::ops::fill_slice(&latents.denoise_mask, + 2, + latents.denoise_mask.shape()[2] - reference_frames, + latents.denoise_mask.shape()[2], + 1.f - reference_strength); + LOG_INFO("LTX IC-LoRA conditioned on %zu static reference image(s), strength %.3f", + reference_images.size(), + reference_strength); + } + int64_t t2 = ggml_time_ms(); LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1); } From 401fc93993640ed889aadfc09355e1e7d125dbaf Mon Sep 17 00:00:00 2001 From: aegioscy Date: Tue, 28 Jul 2026 10:13:08 +0200 Subject: [PATCH 2/4] fix(ltx): align Ingredients reference positions Use causal temporal positions for appended reference tokens and reject unsupported multi-sheet conditioning. --- src/stable-diffusion.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 7fad365d0..2b2f4dc78 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -3717,8 +3717,12 @@ static sd::Tensor build_ltxv_video_positions(int64_t width, } for (int64_t t = 0; t < keyframe_latent_frames; t++) { - float t_start = static_cast(keyframe_frame_idx + t * temporal_scale); - float t_end = static_cast(keyframe_frame_idx + (t + 1) * temporal_scale); + const int64_t start_corner = keyframe_frame_idx + t; + const int64_t end_corner = keyframe_frame_idx + t + 1; + float t_start = ltxv_latent_corner_to_pixel_frame( + start_corner, temporal_scale, causal_temporal_positioning); + float t_end = ltxv_latent_corner_to_pixel_frame( + end_corner, temporal_scale, causal_temporal_positioning); if (keyframe_pixel_frames == 1) { t_end = t_start + 1.f; } @@ -4887,6 +4891,10 @@ static std::optional prepare_video_generation_latents(sd return std::nullopt; } if (!reference_images.empty()) { + if (reference_images.size() != 1) { + LOG_ERROR("LTX Ingredients requires exactly one composite reference sheet"); + return std::nullopt; + } if (sd_vid_gen_params->reference_downscale_factor != 1.f) { LOG_ERROR("LTX IC-LoRA currently requires reference_downscale_factor=1"); return std::nullopt; From 146ede0c888b5efa7db9521aefd4164a42651242 Mon Sep 17 00:00:00 2001 From: aegioscy Date: Tue, 28 Jul 2026 17:26:05 +0200 Subject: [PATCH 3/4] fix(ltx): align IC-LoRA inference workflow Match official reference-first conditioning and add video-only spatiotemporal guidance so Ingredients generation preserves reference identity and motion quality. --- src/diffusion_model.hpp | 1 + src/ltxv.hpp | 85 ++++++++++++++++++++------------ src/stable-diffusion.cpp | 104 +++++++++++++++++++++------------------ 3 files changed, 112 insertions(+), 78 deletions(-) diff --git a/src/diffusion_model.hpp b/src/diffusion_model.hpp index 80c115449..bde899e76 100644 --- a/src/diffusion_model.hpp +++ b/src/diffusion_model.hpp @@ -47,6 +47,7 @@ struct LTXAVDiffusionExtra { int audio_length = 0; float frame_rate = 24.f; const sd::Tensor* video_positions = nullptr; + const std::vector* skip_video_self_attention_blocks = nullptr; }; using DiffusionExtraParams = std::variant(blocks["attn1"]); auto audio_attn1 = std::dynamic_pointer_cast(blocks["audio_attn1"]); auto attn2 = std::dynamic_pointer_cast(blocks["attn2"]); @@ -1089,10 +1090,12 @@ namespace LTXV { bool run_v2a = run_ax; auto v_mods = get_ada_values(ctx, v_table, v_timestep, v_dim, cross_attention_adaln ? 9 : 6); - auto v_norm = rms_norm(ctx->ggml_ctx, vx); - v_norm = modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]); - auto v_sa = attn1->forward(ctx, v_norm, nullptr, self_attention_mask, v_pe); - vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_sa, v_mods[2])); + if (!skip_video_self_attention) { + auto v_norm = rms_norm(ctx->ggml_ctx, vx); + v_norm = modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]); + auto v_sa = attn1->forward(ctx, v_norm, nullptr, self_attention_mask, v_pe); + vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_sa, v_mods[2])); + } auto v_txt = apply_text_cross_attention(ctx, vx, v_context, @@ -1418,7 +1421,8 @@ namespace LTXV { ggml_tensor* v_cross_pe, ggml_tensor* a_cross_pe, ggml_tensor* video_connector_pe, - ggml_tensor* audio_connector_pe) { + ggml_tensor* audio_connector_pe, + const std::vector* skip_video_self_attention_blocks = nullptr) { auto patchify_proj = std::dynamic_pointer_cast(blocks["patchify_proj"]); auto audio_patchify_proj = std::dynamic_pointer_cast(blocks["audio_patchify_proj"]); auto adaln_single = std::dynamic_pointer_cast(blocks["adaln_single"]); @@ -1493,26 +1497,33 @@ namespace LTXV { for (int i = 0; i < cfg.num_layers; i++) { auto block = std::dynamic_pointer_cast(blocks["transformer_blocks." + std::to_string(i)]); - auto out = block->forward(ctx, - vx, - ax, - v_context, - a_context, - nullptr, - v_timestep_mod, - a_timestep_mod, - v_pe, - a_pe, - v_cross_pe, - a_cross_pe, - av_ca_video_scale_shift_timestep, - av_ca_audio_scale_shift_timestep, - av_ca_a2v_gate_noise_timestep, - av_ca_v2a_gate_noise_timestep, - v_prompt_timestep_mod, - a_prompt_timestep_mod); - vx = out.first; - ax = out.second; + const bool skip_video_self_attention = + skip_video_self_attention_blocks != nullptr && + std::find(skip_video_self_attention_blocks->begin(), + skip_video_self_attention_blocks->end(), + i) != skip_video_self_attention_blocks->end(); + auto out = block->forward(ctx, + vx, + ax, + v_context, + a_context, + nullptr, + v_timestep_mod, + a_timestep_mod, + v_pe, + a_pe, + v_cross_pe, + a_cross_pe, + av_ca_video_scale_shift_timestep, + av_ca_audio_scale_shift_timestep, + av_ca_a2v_gate_noise_timestep, + av_ca_v2a_gate_noise_timestep, + v_prompt_timestep_mod, + a_prompt_timestep_mod, + nullptr, + skip_video_self_attention); + vx = out.first; + ax = out.second; sd::ggml_graph_cut::mark_graph_cut(vx, "ltxav.transformer_blocks." + std::to_string(i), "vx"); sd::ggml_graph_cut::mark_graph_cut(ax, "ltxav.transformer_blocks." + std::to_string(i), "ax"); } @@ -1744,7 +1755,8 @@ namespace LTXV { const sd::Tensor& audio_timesteps_tensor = {}, int audio_length = 0, float frame_rate = 24.f, - const sd::Tensor& video_positions_tensor = {}) { + const sd::Tensor& video_positions_tensor = {}, + const std::vector* skip_video_self_attention_blocks = nullptr) { auto split_inputs = split_av_latents(x_tensor, audio_length); vx_input_cache = split_inputs.first; if (!audio_x_tensor.empty()) { @@ -1894,7 +1906,8 @@ namespace LTXV { video_cross_pe, audio_cross_pe, video_connector_pe, - audio_connector_pe); + audio_connector_pe, + skip_video_self_attention_blocks); auto out = merge_av_latents(compute_ctx, out_pair.first, out_pair.second); ggml_build_forward_expand(gf, out); return gf; @@ -1908,9 +1921,18 @@ namespace LTXV { const sd::Tensor& audio_timesteps = {}, int audio_length = 0, float frame_rate = 24.f, - const sd::Tensor& video_positions = {}) { + const sd::Tensor& video_positions = {}, + const std::vector* skip_video_self_attention_blocks = nullptr) { auto get_graph = [&]() -> ggml_cgraph* { - return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions); + return build_graph(x, + timesteps, + context, + audio_x, + audio_timesteps, + audio_length, + frame_rate, + video_positions, + skip_video_self_attention_blocks); }; auto out = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); return out; @@ -1929,7 +1951,8 @@ namespace LTXV { tensor_or_empty(extra->audio_timesteps), extra->audio_length, extra->frame_rate, - tensor_or_empty(extra->video_positions)); + tensor_or_empty(extra->video_positions), + extra->skip_video_self_attention_blocks); } void test(const std::string& x_path, diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 2b2f4dc78..f3d12ac7a 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2207,7 +2207,8 @@ class StableDiffusionGGML { audio_timesteps_tensor.empty() ? nullptr : &audio_timesteps_tensor, audio_length, frame_rate, - video_positions.empty() ? nullptr : &video_positions}; + video_positions.empty() ? nullptr : &video_positions, + local_skip_layers}; } else { diffusion_params.extra = std::monostate{}; } @@ -3655,6 +3656,7 @@ struct ImageGenerationLatents { int64_t ref_image_num = 0; int64_t video_conditioning_frame_count = 0; int64_t video_target_frame_count = 0; + bool video_conditioning_first = false; int audio_length = 0; }; @@ -3693,7 +3695,8 @@ static sd::Tensor build_ltxv_video_positions(int64_t width, int fps, int spatial_scale, int temporal_scale, - bool causal_temporal_positioning) { + bool causal_temporal_positioning, + bool keyframe_first = false) { GGML_ASSERT(width > 0 && height > 0 && target_latent_frames > 0); GGML_ASSERT(keyframe_latent_frames > 0); GGML_ASSERT(fps > 0); @@ -3702,41 +3705,38 @@ static sd::Tensor build_ltxv_video_positions(int64_t width, sd::Tensor positions({2, 3, total_tokens, 1}); int64_t token = 0; - for (int64_t t = 0; t < target_latent_frames; t++) { - float t_start = ltxv_latent_corner_to_pixel_frame(t, temporal_scale, causal_temporal_positioning) / static_cast(fps); - float t_end = ltxv_latent_corner_to_pixel_frame(t + 1, temporal_scale, causal_temporal_positioning) / static_cast(fps); - for (int64_t h = 0; h < height; h++) { - float h_start = static_cast(h * spatial_scale); - float h_end = static_cast((h + 1) * spatial_scale); - for (int64_t w = 0; w < width; w++) { - float w_start = static_cast(w * spatial_scale); - float w_end = static_cast((w + 1) * spatial_scale); - set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end); + auto append_positions = [&](bool keyframe) { + const int64_t frame_count = keyframe ? keyframe_latent_frames : target_latent_frames; + for (int64_t t = 0; t < frame_count; t++) { + const int64_t start_corner = (keyframe ? keyframe_frame_idx : 0) + t; + const int64_t end_corner = start_corner + 1; + float t_start = ltxv_latent_corner_to_pixel_frame( + start_corner, temporal_scale, causal_temporal_positioning); + float t_end = ltxv_latent_corner_to_pixel_frame( + end_corner, temporal_scale, causal_temporal_positioning); + if (keyframe && keyframe_pixel_frames == 1) { + t_end = t_start + 1.f; + } + t_start /= static_cast(fps); + t_end /= static_cast(fps); + for (int64_t h = 0; h < height; h++) { + float h_start = static_cast(h * spatial_scale); + float h_end = static_cast((h + 1) * spatial_scale); + for (int64_t w = 0; w < width; w++) { + float w_start = static_cast(w * spatial_scale); + float w_end = static_cast((w + 1) * spatial_scale); + set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end); + } } } - } + }; - for (int64_t t = 0; t < keyframe_latent_frames; t++) { - const int64_t start_corner = keyframe_frame_idx + t; - const int64_t end_corner = keyframe_frame_idx + t + 1; - float t_start = ltxv_latent_corner_to_pixel_frame( - start_corner, temporal_scale, causal_temporal_positioning); - float t_end = ltxv_latent_corner_to_pixel_frame( - end_corner, temporal_scale, causal_temporal_positioning); - if (keyframe_pixel_frames == 1) { - t_end = t_start + 1.f; - } - t_start /= static_cast(fps); - t_end /= static_cast(fps); - for (int64_t h = 0; h < height; h++) { - float h_start = static_cast(h * spatial_scale); - float h_end = static_cast((h + 1) * spatial_scale); - for (int64_t w = 0; w < width; w++) { - float w_start = static_cast(w * spatial_scale); - float w_end = static_cast((w + 1) * spatial_scale); - set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end); - } - } + if (keyframe_first) { + append_positions(true); + append_positions(false); + } else { + append_positions(false); + append_positions(true); } return positions; @@ -4931,7 +4931,8 @@ static std::optional prepare_video_generation_latents(sd auto apply_video_condition_by_keyframe_index = [&](const sd::Tensor& keyframes, int frame_idx, - const char* name) -> bool { + const char* name, + bool condition_first = false) -> bool { int64_t keyframe_frames = keyframes.shape()[2]; if (keyframe_frames <= 0 || keyframes.shape()[0] != latents.init_latent.shape()[0] || keyframes.shape()[1] != latents.init_latent.shape()[1] || @@ -4942,15 +4943,21 @@ static std::optional prepare_video_generation_latents(sd latents.video_target_frame_count = latents.init_latent.shape()[2]; latents.video_conditioning_frame_count = keyframe_frames; - latents.init_latent = sd::ops::concat(latents.init_latent, keyframes, 2); + latents.video_conditioning_first = condition_first; - auto keyframe_mask = sd::full({keyframes.shape()[0], - keyframes.shape()[1], - keyframes.shape()[2], - 1, - 1}, + auto keyframe_mask = sd::full({keyframes.shape()[0], + keyframes.shape()[1], + keyframes.shape()[2], + 1, + 1}, conditioned_mask); - latents.denoise_mask = sd::ops::concat(latents.denoise_mask, keyframe_mask, 2); + if (condition_first) { + latents.init_latent = sd::ops::concat(keyframes, latents.init_latent, 2); + latents.denoise_mask = sd::ops::concat(keyframe_mask, latents.denoise_mask, 2); + } else { + latents.init_latent = sd::ops::concat(latents.init_latent, keyframes, 2); + latents.denoise_mask = sd::ops::concat(latents.denoise_mask, keyframe_mask, 2); + } latents.video_positions = build_ltxv_video_positions(latents.init_latent.shape()[0], latents.init_latent.shape()[1], latents.video_target_frame_count, @@ -4960,7 +4967,8 @@ static std::optional prepare_video_generation_latents(sd request->fps, request->vae_scale_factor, 8, - true); + true, + condition_first); return true; }; @@ -5011,7 +5019,8 @@ static std::optional prepare_video_generation_latents(sd } if (!apply_video_condition_by_keyframe_index(reference_latents, 0, - "IC-LoRA reference")) { + "IC-LoRA reference", + true)) { return std::nullopt; } const float reference_strength = @@ -5019,8 +5028,8 @@ static std::optional prepare_video_generation_latents(sd const int64_t reference_frames = reference_latents.shape()[2]; sd::ops::fill_slice(&latents.denoise_mask, 2, - latents.denoise_mask.shape()[2] - reference_frames, - latents.denoise_mask.shape()[2], + 0, + reference_frames, 1.f - reference_strength); LOG_INFO("LTX IC-LoRA conditioned on %zu static reference image(s), strength %.3f", reference_images.size(), @@ -5828,7 +5837,8 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, if (latents.video_conditioning_frame_count > 0) { int64_t target_frames = latents.video_target_frame_count > 0 ? latents.video_target_frame_count : final_latent.shape()[2] - latents.video_conditioning_frame_count; - final_latent = sd::ops::slice(final_latent, 2, 0, target_frames); + int64_t target_start = latents.video_conditioning_first ? latents.video_conditioning_frame_count : 0; + final_latent = sd::ops::slice(final_latent, 2, target_start, target_start + target_frames); } if (latents.ref_image_num > 0) { From 6ecf33061849e0bc125a604d83b883a32f8d3e92 Mon Sep 17 00:00:00 2001 From: aegioscy Date: Tue, 4 Aug 2026 14:44:58 +0300 Subject: [PATCH 4/4] fix(ltx): stabilize RoPE and projection precision Use shared host-generated RoPE tables and request full-F32 patch projections to reduce backend-dependent drift. --- CMakeLists.txt | 7 +- src/ltxv.hpp | 165 ++++++++++++++++------------------------ tests/CMakeLists.txt | 3 + tests/test-ltx-rope.cpp | 65 ++++++++++++++++ 4 files changed, 138 insertions(+), 102 deletions(-) create mode 100644 tests/CMakeLists.txt create mode 100644 tests/test-ltx-rope.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 242b84292..82798380d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,7 +83,7 @@ endif() # # general -#option(SD_BUILD_TESTS "sd: build tests" ${SD_STANDALONE}) +option(SD_BUILD_TESTS "sd: build tests" OFF) option(SD_BUILD_EXAMPLES "sd: build examples" ${SD_STANDALONE}) option(SD_WEBP "sd: enable WebP image I/O support" ${SD_WEBP_DEFAULT}) option(SD_USE_SYSTEM_WEBP "sd: link against system libwebp" OFF) @@ -340,6 +340,11 @@ if (SD_BUILD_EXAMPLES) add_subdirectory(examples) endif() +if (SD_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif() + set(SD_PUBLIC_HEADERS include/stable-diffusion.h) set_target_properties(${SD_LIB} PROPERTIES PUBLIC_HEADER "${SD_PUBLIC_HEADERS}") diff --git a/src/ltxv.hpp b/src/ltxv.hpp index a4bca989b..e1ad4df52 100644 --- a/src/ltxv.hpp +++ b/src/ltxv.hpp @@ -72,33 +72,12 @@ namespace LTXV { return max_block + 1; } - __STATIC_INLINE__ std::vector generate_freq_grid(float theta, - int positional_dims, - int dim) { - const int n_elem = 2 * positional_dims; - const int freq_count = dim / n_elem; - - std::vector out(freq_count); - if (freq_count <= 0) { - return out; - } - if (freq_count == 1) { - out[0] = 1.5707963267948966f; - return out; - } - - const float half_pi = 1.5707963267948966f; - const float log_theta = std::log(theta); - for (int i = 0; i < freq_count; i++) { - float ratio = static_cast(i) / static_cast(freq_count - 1); - out[i] = std::exp(log_theta * ratio) * half_pi; - } - return out; - } - - __STATIC_INLINE__ std::vector generate_freq_grid_double(double theta, - int positional_dims, - int dim) { + // Prepare RoPE matrices once on the host in binary64, then round the shared + // table to F32 before copying it to any backend. This avoids backend trig + // implementations and single-precision intermediate differences. + __STATIC_INLINE__ std::vector generate_freq_grid(double theta, + int positional_dims, + int dim) { const int n_elem = 2 * positional_dims; const int freq_count = dim / n_elem; @@ -107,30 +86,30 @@ namespace LTXV { return out; } if (freq_count == 1) { - out[0] = 1.5707963267948966; + out[0] = 0x1.921fb54442d18p+0; return out; } - const double half_pi = 1.5707963267948966; + const double half_pi = 0x1.921fb54442d18p+0; const double log_theta = std::log(theta); for (int i = 0; i < freq_count; i++) { double ratio = static_cast(i) / static_cast(freq_count - 1); - out[i] = std::exp(log_theta * ratio) * half_pi; + out[i] = std::exp(log_theta * ratio) * half_pi; } return out; } __STATIC_INLINE__ std::vector build_rope_matrix_from_frequencies( - const std::vector>& frequencies, + const std::vector>& frequencies, int dim) { const int half_dim = dim / 2; std::vector out(static_cast(frequencies.size()) * static_cast(half_dim) * 4, 0.f); for (size_t token = 0; token < frequencies.size(); token++) { for (int i = 0; i < half_dim; i++) { - float angle = i < static_cast(frequencies[token].size()) ? frequencies[token][i] : 0.f; - float c = std::cos(angle); - float s = std::sin(angle); + double angle = i < static_cast(frequencies[token].size()) ? frequencies[token][i] : 0.0; + float c = static_cast(std::cos(angle)); + float s = static_cast(std::sin(angle)); size_t base = (token * static_cast(half_dim) + static_cast(i)) * 4; out[base + 0] = c; @@ -143,8 +122,8 @@ namespace LTXV { return out; } - __STATIC_INLINE__ std::vector> split_frequencies_by_heads( - const std::vector>& frequencies, + __STATIC_INLINE__ std::vector> split_frequencies_by_heads( + const std::vector>& frequencies, int inner_dim, int num_heads) { GGML_ASSERT(num_heads > 0); @@ -153,9 +132,9 @@ namespace LTXV { const int per_head_half_dim = inner_half_dim / num_heads; GGML_ASSERT(inner_half_dim % num_heads == 0); - std::vector> out( + std::vector> out( frequencies.size() * static_cast(num_heads), - std::vector(per_head_half_dim, 0.f)); + std::vector(per_head_half_dim, 0.0)); for (size_t token = 0; token < frequencies.size(); token++) { GGML_ASSERT(static_cast(frequencies[token].size()) == inner_half_dim); @@ -180,11 +159,11 @@ namespace LTXV { bool use_middle_indices_grid = false) { GGML_ASSERT(max_pos.size() == 3); GGML_ASSERT(dim % num_heads == 0); - const std::vector indices = generate_freq_grid(theta, 3, dim); + const std::vector indices = generate_freq_grid(static_cast(theta), 3, dim); const int half_dim = dim / 2; const int pad_size = half_dim - static_cast(indices.size()) * 3; - std::vector> freqs(static_cast(width * height * frames), std::vector(half_dim, 0.f)); + std::vector> freqs(static_cast(width * height * frames), std::vector(half_dim, 0.0)); const int scale_t = std::get<0>(vae_scale_factors); const int scale_h = std::get<1>(vae_scale_factors); @@ -192,45 +171,45 @@ namespace LTXV { size_t token = 0; for (int64_t t = 0; t < frames; t++) { - float pixel_t = static_cast(t * scale_t); + double pixel_t = static_cast(t * scale_t); if (causal_temporal_positioning) { - pixel_t = std::max(0.f, pixel_t + 1.f - scale_t); + pixel_t = std::max(0.0, pixel_t + 1.0 - scale_t); } - pixel_t /= frame_rate; + pixel_t /= static_cast(frame_rate); if (use_middle_indices_grid) { - float end = static_cast((t + 1) * scale_t); + double end = static_cast((t + 1) * scale_t); if (causal_temporal_positioning) { - end = std::max(0.f, end + 1.f - scale_t); + end = std::max(0.0, end + 1.0 - scale_t); } - end /= frame_rate; - pixel_t = 0.5f * (pixel_t + end); + end /= static_cast(frame_rate); + pixel_t = 0.5 * (pixel_t + end); } for (int64_t h = 0; h < height; h++) { - float pixel_h = static_cast(h * scale_h); + double pixel_h = static_cast(h * scale_h); if (use_middle_indices_grid) { - pixel_h += 0.5f * static_cast(scale_h); + pixel_h += 0.5 * static_cast(scale_h); } for (int64_t w = 0; w < width; w++) { - float pixel_w = static_cast(w * scale_w); + double pixel_w = static_cast(w * scale_w); if (use_middle_indices_grid) { - pixel_w += 0.5f * static_cast(scale_w); + pixel_w += 0.5 * static_cast(scale_w); } int out_idx = 0; for (int i = 0; i < pad_size; i++) { - freqs[token][out_idx++] = 0.f; + freqs[token][out_idx++] = 0.0; } - const float coords[3] = { + const double coords[3] = { pixel_t / max_pos[0], pixel_h / max_pos[1], pixel_w / max_pos[2], }; - for (float index : indices) { + for (double index : indices) { for (int axis = 0; axis < 3; axis++) { - freqs[token][out_idx++] = index * (coords[axis] * 2.f - 1.f); + freqs[token][out_idx++] = index * (coords[axis] * 2.0 - 1.0); } } token++; @@ -260,30 +239,30 @@ namespace LTXV { } const int64_t tokens = positions.shape()[2]; - const std::vector indices = generate_freq_grid(theta, 3, dim); + const std::vector indices = generate_freq_grid(static_cast(theta), 3, dim); const int half_dim = dim / 2; const int pad_size = half_dim - static_cast(indices.size()) * 3; - std::vector> freqs(static_cast(tokens), std::vector(half_dim, 0.f)); + std::vector> freqs(static_cast(tokens), std::vector(half_dim, 0.0)); for (int64_t token = 0; token < tokens; token++) { int out_idx = 0; for (int i = 0; i < pad_size; i++) { - freqs[token][out_idx++] = 0.f; + freqs[token][out_idx++] = 0.0; } - float coords[3]; + double coords[3]; for (int axis = 0; axis < 3; axis++) { - float start = positions.dim() == 4 ? positions.index(0, axis, token, 0) - : positions.index(0, axis, token); - float end = positions.dim() == 4 ? positions.index(1, axis, token, 0) - : positions.index(1, axis, token); - float coord = use_middle_indices_grid ? 0.5f * (start + end) : start; - coords[axis] = coord / static_cast(max_pos[axis]); + double start = positions.dim() == 4 ? static_cast(positions.index(0, axis, token, 0)) + : static_cast(positions.index(0, axis, token)); + double end = positions.dim() == 4 ? static_cast(positions.index(1, axis, token, 0)) + : static_cast(positions.index(1, axis, token)); + double coord = use_middle_indices_grid ? 0.5 * (start + end) : start; + coords[axis] = coord / static_cast(max_pos[axis]); } - for (float index : indices) { + for (double index : indices) { for (int axis = 0; axis < 3; axis++) { - freqs[token][out_idx++] = index * (coords[axis] * 2.f - 1.f); + freqs[token][out_idx++] = index * (coords[axis] * 2.0 - 1.0); } } } @@ -301,29 +280,21 @@ namespace LTXV { float positional_scale = 4096.f, bool double_precision = false) { GGML_ASSERT(dim % num_heads == 0); - const std::vector indices = double_precision ? std::vector() : generate_freq_grid(theta, 1, dim); - const std::vector indices_d = - double_precision ? generate_freq_grid_double(static_cast(theta), 1, dim) : std::vector(); + (void) double_precision; // Kept for source compatibility; preparation is always binary64. + const std::vector indices = generate_freq_grid(static_cast(theta), 1, dim); const int half_dim = dim / 2; - const int pad_size = half_dim - static_cast(double_precision ? indices_d.size() : indices.size()); + const int pad_size = half_dim - static_cast(indices.size()); - std::vector> freqs(static_cast(seq_len), std::vector(half_dim, 0.f)); + std::vector> freqs(static_cast(seq_len), std::vector(half_dim, 0.0)); for (int64_t pos = 0; pos < seq_len; pos++) { int out_idx = 0; for (int i = 0; i < pad_size; i++) { - freqs[static_cast(pos)][out_idx++] = 0.f; + freqs[static_cast(pos)][out_idx++] = 0.0; } - if (double_precision) { - double coord = static_cast(pos) / static_cast(positional_scale); - for (double index : indices_d) { - freqs[static_cast(pos)][out_idx++] = static_cast(index * (coord * 2.0 - 1.0)); - } - } else { - float coord = static_cast(pos) / positional_scale; - for (float index : indices) { - freqs[static_cast(pos)][out_idx++] = index * (coord * 2.f - 1.f); - } + double coord = static_cast(pos) / static_cast(positional_scale); + for (double index : indices) { + freqs[static_cast(pos)][out_idx++] = index * (coord * 2.0 - 1.0); } } @@ -830,28 +801,20 @@ namespace LTXV { float max_pos = 20.f, bool double_precision = false) { GGML_ASSERT(dim % num_heads == 0); - const std::vector indices = double_precision ? std::vector() : generate_freq_grid(theta, 1, dim); - const std::vector indices_d = - double_precision ? generate_freq_grid_double(static_cast(theta), 1, dim) : std::vector(); + (void) double_precision; // Kept for source compatibility; preparation is always binary64. + const std::vector indices = generate_freq_grid(static_cast(theta), 1, dim); const int half_dim = dim / 2; - const int pad_size = half_dim - static_cast(double_precision ? indices_d.size() : indices.size()); + const int pad_size = half_dim - static_cast(indices.size()); - std::vector> freqs(coords.size(), std::vector(half_dim, 0.f)); + std::vector> freqs(coords.size(), std::vector(half_dim, 0.0)); for (size_t pos = 0; pos < coords.size(); pos++) { int out_idx = 0; for (int i = 0; i < pad_size; i++) { - freqs[pos][out_idx++] = 0.f; + freqs[pos][out_idx++] = 0.0; } - if (double_precision) { - double coord = static_cast(coords[pos]) / static_cast(max_pos); - for (double index : indices_d) { - freqs[pos][out_idx++] = static_cast(index * (coord * 2.0 - 1.0)); - } - } else { - float coord = coords[pos] / max_pos; - for (float index : indices) { - freqs[pos][out_idx++] = index * (coord * 2.f - 1.f); - } + double coord = static_cast(coords[pos]) / static_cast(max_pos); + for (double index : indices) { + freqs[pos][out_idx++] = index * (coord * 2.0 - 1.0); } } if (num_heads > 1) { @@ -1189,8 +1152,8 @@ namespace LTXV { LTXAVModelBlock(const LTXAVParams& params) : cfg(params) { - blocks["patchify_proj"] = std::make_shared(cfg.in_channels, cfg.hidden_size, true, true); - blocks["audio_patchify_proj"] = std::make_shared(cfg.audio_in_channels, cfg.audio_hidden_size, true, true); + blocks["patchify_proj"] = std::make_shared(cfg.in_channels, cfg.hidden_size, true, true, true); + blocks["audio_patchify_proj"] = std::make_shared(cfg.audio_in_channels, cfg.audio_hidden_size, true, true, true); blocks["adaln_single"] = std::make_shared(cfg.hidden_size, cfg.cross_attention_adaln ? 9 : 6); blocks["audio_adaln_single"] = std::make_shared(cfg.audio_hidden_size, cfg.cross_attention_adaln ? 9 : 6); if (cfg.cross_attention_adaln) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..b4e1bc543 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(test-ltx-rope test-ltx-rope.cpp) +target_link_libraries(test-ltx-rope PRIVATE stable-diffusion) +add_test(NAME test-ltx-rope COMMAND test-ltx-rope) diff --git a/tests/test-ltx-rope.cpp b/tests/test-ltx-rope.cpp new file mode 100644 index 000000000..9eccdf338 --- /dev/null +++ b/tests/test-ltx-rope.cpp @@ -0,0 +1,65 @@ +#include "ltxv.hpp" + +#include +#include +#include +#include +#include + +static uint64_t hash_float_bits(const std::vector& values) { + uint64_t hash = 1469598103934665603ULL; + for (float value : values) { + uint32_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + for (int shift = 0; shift < 32; shift += 8) { + hash ^= (bits >> shift) & 0xffU; + hash *= 1099511628211ULL; + } + } + return hash; +} + +static bool has_valid_rotation_blocks(const std::vector& values) { + if (values.size() % 4 != 0) { + return false; + } + for (size_t i = 0; i < values.size(); i += 4) { + if (values[i] != values[i + 3] || values[i + 1] != -values[i + 2]) { + return false; + } + const double norm = static_cast(values[i]) * values[i] + + static_cast(values[i + 2]) * values[i + 2]; + if (std::abs(norm - 1.0) > 1e-7) { + return false; + } + } + return true; +} + +int main() { + const auto video = LTXV::build_video_rope_matrix( + 4, 3, 2, 4096, 32, 24.f, 10000.f, {20, 2048, 2048}, {8, 32, 32}, false, true); + const auto connector = LTXV::build_1d_rope_matrix(37, 1024, 8, 10000.f, 4096.f, true); + + const uint64_t video_hash = hash_float_bits(video); + const uint64_t connector_hash = hash_float_bits(connector); + + if (!has_valid_rotation_blocks(video) || !has_valid_rotation_blocks(connector)) { + std::fprintf(stderr, "invalid RoPE rotation block\n"); + return 1; + } + + // Golden hashes cover frequency generation, coordinate normalization, + // head splitting, and binary64 sin/cos rounding to the shared F32 table. + constexpr uint64_t expected_video_hash = 0x17060c3ec6c902ebULL; + constexpr uint64_t expected_connector_hash = 0x89d227927a382d13ULL; + if (video_hash != expected_video_hash || connector_hash != expected_connector_hash) { + std::fprintf(stderr, + "RoPE hash mismatch: video=%016llx connector=%016llx\n", + static_cast(video_hash), + static_cast(connector_hash)); + return 1; + } + + return 0; +}