From f5745dd7ff39809e8ae48d518e2db64d8dc3227f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zsombor=20Par=C3=B3czi?= Date: Tue, 25 Aug 2026 23:08:38 +0200 Subject: [PATCH] layer split: make the compute headroom overridable, and say what actually failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2 GiB compute headroom held back on every device was hardcoded with no override, and on an asymmetric multi-GPU pair that is what makes layer split unusable. Measured on 8 GB + 12 GB with MiniMax-H3 ref2va Q4 at 1344x768: the params pass fills the 8 GB card to 5473 MB, leaving 2719 MB free, and 2 GiB of that is then reserved — so the small card absorbs no graph segments at all, everything piles onto the 12 GB card, and the plan is refused by 165 MB. --max-vram cannot rescue it: that limit is applied with min(), so it can only ever lower the capacity, never raise it. Every value from 7.5 to 11.7 GB gave byte-identical results because the free-VRAM term always won. SD_COMPUTE_HEADROOM_MB makes the trade — refuse-to-plan versus risking a real OOM — a per-host decision. Default unchanged at 2 GiB. The error message also reported `current_used + bytes` as what the segment "needs", which reads as one huge segment. It is cumulative, so it does not shrink when you lower the resolution: I chased a smaller canvas from 1344x768 to 1280x736 and the number did not move a single byte. It now prints the segment's own size, what was already packed, the capacity and the shortfall. --- src/core/layer_split_partition.cpp | 45 +++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/core/layer_split_partition.cpp b/src/core/layer_split_partition.cpp index 654b3056b..744b52ea4 100644 --- a/src/core/layer_split_partition.cpp +++ b/src/core/layer_split_partition.cpp @@ -77,7 +77,30 @@ namespace sd { const std::vector& backend_vram_limits, size_t primary_backend_vram_limit) { std::vector capacities(backends.size(), std::numeric_limits::max() / 4); - constexpr int64_t compute_headroom_bytes = 2ll * 1024 * 1024 * 1024; + // The headroom held back on every device for the compute buffer. + // + // It was a hardcoded 2 GiB with no override, and on an asymmetric pair + // that is what makes multi-GPU layer split unusable. Measured on an + // 8 GB + 12 GB pair with MiniMax-H3 ref2va Q4: the params pass fills the + // 8 GB card to 5473 MB, leaving 2719 MB free, and 2 GiB of that is then + // reserved — so the smaller card can absorb almost no graph segments and + // everything piles onto the larger one, which then refuses the graph by + // 165 MB. Note also that the --max-vram limit below is a min(), so no + // value of it can ever RAISE this; the free-VRAM term always wins. + // + // Overridable so the trade (refuse-to-plan vs risk a real OOM) can be + // made per host without a rebuild. Default unchanged at 2 GiB. + int64_t compute_headroom_bytes = 2ll * 1024 * 1024 * 1024; + if (const char* env = getenv("SD_COMPUTE_HEADROOM_MB")) { + char* end = nullptr; + long long value = strtoll(env, &end, 10); + if (end != env && value >= 0) { + compute_headroom_bytes = (int64_t)value * 1024 * 1024; + LOG_DEBUG("graph-cut layer split: compute headroom overridden to %lld MB", value); + } else { + LOG_WARN("SD_COMPUTE_HEADROOM_MB is not a non-negative integer, ignoring: %s", env); + } + } for (size_t i = 0; i < backends.size(); i++) { ggml_backend_dev_t dev = ggml_backend_get_device(backends[i]); size_t free_bytes = 0, total_bytes = 0; @@ -158,12 +181,26 @@ namespace sd { current_used = 0; } if (bytes > 0 && current_used + bytes > backend_capacities[current_backend]) { - LOG_ERROR("%s graph-cut layer split: segment %zu needs %.1f MB on %s, but only %.1f MB is available under current VRAM limits", + // Report the segment's OWN size and the running total separately. + // Printing only `current_used + bytes` as what the segment + // "needs" reads as a single huge segment and sends you looking + // for a smaller canvas — but this figure is cumulative, so it + // does not shrink with resolution and the search goes nowhere. + // The actionable numbers are: how big this segment is, how much + // was already packed onto this device, and the shortfall. + LOG_ERROR("%s graph-cut layer split: segment %zu (%.1f MB) does not fit %s: " + "%.1f MB already packed + %.1f MB > %.1f MB capacity (short by %.1f MB). " + "Capacity is min(free VRAM - compute headroom, --max-vram), so --max-vram " + "cannot raise it; lower SD_COMPUTE_HEADROOM_MB (default 2048) or free VRAM " + "on that device", desc, seg_idx, - (current_used + bytes) / (1024.0 * 1024.0), + bytes / (1024.0 * 1024.0), layer_split_backend_device_display_name(split_backends[current_backend]).c_str(), - backend_capacities[current_backend] / (1024.0 * 1024.0)); + current_used / (1024.0 * 1024.0), + bytes / (1024.0 * 1024.0), + backend_capacities[current_backend] / (1024.0 * 1024.0), + (current_used + bytes - backend_capacities[current_backend]) / (1024.0 * 1024.0)); return false; } current_used += bytes;