diff --git a/src/model/diffusion/minimax_h3.hpp b/src/model/diffusion/minimax_h3.hpp index d0683166d..84f37ebed 100644 --- a/src/model/diffusion/minimax_h3.hpp +++ b/src/model/diffusion/minimax_h3.hpp @@ -130,6 +130,18 @@ namespace MiniMaxH3 { return to_shift * a * a / (from_shift * b * b); } + static float time_shift_step_scale(float sigma, + float next_sigma, + float from_shift, + float to_shift) { + if (!std::isfinite(next_sigma) || next_sigma < 0.f || next_sigma == sigma) { + return time_shift_slope(sigma, from_shift, to_shift); + } + float shifted_sigma = time_shift_sigma(sigma, from_shift, to_shift); + float shifted_next_sigma = time_shift_sigma(next_sigma, from_shift, to_shift); + return (shifted_sigma - shifted_next_sigma) / (sigma - next_sigma); + } + struct TimeEmbedder : public GGMLBlock { TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) { blocks["proj_in"] = std::make_shared(input_dim, hidden_dim, true, true); @@ -1033,7 +1045,8 @@ namespace MiniMaxH3 { const std::vector& reference_blocks, int audio_length, float video_shift, - float audio_shift) { + float audio_shift, + float next_video_sigma) { auto split = split_av_latents(packed, audio_length); video_input_cache = std::move(split.first); audio_input_cache = std::move(split.second); @@ -1130,7 +1143,16 @@ namespace MiniMaxH3 { layout.sequence_segments, layout.video_segment, layout.audio_segment, - time_shift_slope(sigma_v, video_shift, audio_shift)); + // The generic Euler sampler advances the packed tensor by + // `next_video_sigma - sigma_v`. For that sampler, scale H3's + // audio velocity by the exact ratio of the independent audio + // step. The derivative approximation substantially oversteps + // at low step counts (the Turbo use case). Retain the local + // slope for samplers that make extra/intermediate evaluations. + time_shift_step_scale(sigma_v, + next_video_sigma, + video_shift, + audio_shift)); auto merged = merge_av_latents(compute_ctx, output.first, output.second); auto graph = new_graph_custom(H3_GRAPH_SIZE); ggml_build_forward_expand(graph, merged); @@ -1162,7 +1184,8 @@ namespace MiniMaxH3 { reference_blocks, extra->audio_length, extra->video_sigma_shift, - extra->audio_sigma_shift); + extra->audio_sigma_shift, + extra->next_video_sigma); }; return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index 070ca53d4..918d0b0e5 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -108,6 +108,8 @@ struct MiniMaxH3DiffusionExtra { int audio_length = 0; float video_sigma_shift = 12.f; float audio_sigma_shift = 3.f; + // Negative when the outer sampler is not a single-evaluation Euler step. + float next_video_sigma = -1.f; }; struct MiniT2IDiffusionExtra { diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 109a2a483..9d7b6c8d5 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2761,7 +2761,11 @@ class StableDiffusionGGML { condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks, audio_length, std::isfinite(active_flow_shift) ? active_flow_shift : 12.f, - 3.f}; + 3.f, + method == EULER_SAMPLE_METHOD && step > 0 && + static_cast(step) < sigmas.size() + ? sigmas[step] + : -1.f}; } else if (sd_version_is_ltxav(version)) { diffusion_params.extra = LTXAVDiffusionExtra{ nullptr,