Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions src/model/diffusion/minimax_h3.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,18 @@ namespace MiniMaxH3 {
return to_shift * a * a / (from_shift * b * b);
}

static float time_shift_step_scale(float sigma,
float next_sigma,
float from_shift,
float to_shift) {
if (!std::isfinite(next_sigma) || next_sigma < 0.f || next_sigma == sigma) {
return time_shift_slope(sigma, from_shift, to_shift);
}
float shifted_sigma = time_shift_sigma(sigma, from_shift, to_shift);
float shifted_next_sigma = time_shift_sigma(next_sigma, from_shift, to_shift);
return (shifted_sigma - shifted_next_sigma) / (sigma - next_sigma);
}

struct TimeEmbedder : public GGMLBlock {
TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) {
blocks["proj_in"] = std::make_shared<Linear>(input_dim, hidden_dim, true, true);
Expand Down Expand Up @@ -1033,7 +1045,8 @@ namespace MiniMaxH3 {
const std::vector<MiniMaxH3ReferenceBlock>& reference_blocks,
int audio_length,
float video_shift,
float audio_shift) {
float audio_shift,
float next_video_sigma) {
auto split = split_av_latents(packed, audio_length);
video_input_cache = std::move(split.first);
audio_input_cache = std::move(split.second);
Expand Down Expand Up @@ -1130,7 +1143,16 @@ namespace MiniMaxH3 {
layout.sequence_segments,
layout.video_segment,
layout.audio_segment,
time_shift_slope(sigma_v, video_shift, audio_shift));
// The generic Euler sampler advances the packed tensor by
// `next_video_sigma - sigma_v`. For that sampler, scale H3's
// audio velocity by the exact ratio of the independent audio
// step. The derivative approximation substantially oversteps
// at low step counts (the Turbo use case). Retain the local
// slope for samplers that make extra/intermediate evaluations.
time_shift_step_scale(sigma_v,
next_video_sigma,
video_shift,
audio_shift));
auto merged = merge_av_latents(compute_ctx, output.first, output.second);
auto graph = new_graph_custom(H3_GRAPH_SIZE);
ggml_build_forward_expand(graph, merged);
Expand Down Expand Up @@ -1162,7 +1184,8 @@ namespace MiniMaxH3 {
reference_blocks,
extra->audio_length,
extra->video_sigma_shift,
extra->audio_sigma_shift);
extra->audio_sigma_shift,
extra->next_video_sigma);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
n_threads,
Expand Down
2 changes: 2 additions & 0 deletions src/model/diffusion/model.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ struct MiniMaxH3DiffusionExtra {
int audio_length = 0;
float video_sigma_shift = 12.f;
float audio_sigma_shift = 3.f;
// Negative when the outer sampler is not a single-evaluation Euler step.
float next_video_sigma = -1.f;
};

struct MiniT2IDiffusionExtra {
Expand Down
6 changes: 5 additions & 1 deletion src/stable-diffusion.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2761,7 +2761,11 @@ class StableDiffusionGGML {
condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks,
audio_length,
std::isfinite(active_flow_shift) ? active_flow_shift : 12.f,
3.f};
3.f,
method == EULER_SAMPLE_METHOD && step > 0 &&
static_cast<size_t>(step) < sigmas.size()
? sigmas[step]
: -1.f};
} else if (sd_version_is_ltxav(version)) {
diffusion_params.extra = LTXAVDiffusionExtra{
nullptr,
Expand Down