From e8346c90fa59f8b26e2fc6b4340fa548762873f4 Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 26 Aug 2026 23:45:16 +0800 Subject: [PATCH] feat: load scaled FP8 weights without upfront conversion --- docs/ideogram4.md | 14 ++++++- ggml | 2 +- include/stable-diffusion.h | 11 +++-- src/core/ggml_extend.hpp | 56 +++++++++++++++++-------- src/model/diffusion/ideogram4.hpp | 2 +- src/model_io/safetensors_io.cpp | 10 ++--- src/model_io/tensor_storage.h | 4 +- src/model_loader.cpp | 70 +------------------------------ src/name_conversion.cpp | 5 +++ 9 files changed, 73 insertions(+), 101 deletions(-) diff --git a/docs/ideogram4.md b/docs/ideogram4.md index e519cd4f2..5a4650e49 100644 --- a/docs/ideogram4.md +++ b/docs/ideogram4.md @@ -11,7 +11,19 @@ - Download Qwen3-VL-8B-Instruct - gguf: https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main -## Convert weights +## Use original FP8 weights + +The original Ideogram4 FP8 safetensors can be loaded directly. FP8 tensors stay +at one byte per element in RAM and VRAM. Backends that cannot multiply FP8 +weights directly cast only the active layer to a temporary BF16 tensor during +execution; the loader does not expand the entire checkpoint to BF16. + +Use `ideogram4_fp8.safetensors` and `ideogram4_uncond_fp8.safetensors` directly +with `--diffusion-model` and `--uncond-diffusion-model`, respectively. + +## Optional conversion for quantization + +The following conversion is only needed when creating a quantized GGUF model. fp8 scale -> bf16 diff --git a/ggml b/ggml index 8e800cef2..032b6997d 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 8e800cef2948046cc47f9db6090491c6128ca42c +Subproject commit 032b6997db4c9c75dc85d8d2bb2beec77b1231b0 diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index bab62bac9..6bea445ab 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -136,10 +136,13 @@ enum sd_type_t { // SD_TYPE_IQ4_NL_4_4 = 36, // SD_TYPE_IQ4_NL_4_8 = 37, // SD_TYPE_IQ4_NL_8_8 = 38, - SD_TYPE_MXFP4 = 39, // MXFP4 (1 block) - SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) - SD_TYPE_Q1_0 = 41, - SD_TYPE_COUNT = 42, + SD_TYPE_MXFP4 = 39, // MXFP4 (1 block) + SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + SD_TYPE_Q1_0 = 41, + SD_TYPE_Q2_0 = 42, + SD_TYPE_F8_E4M3 = 43, + SD_TYPE_F8_E5M2 = 44, + SD_TYPE_COUNT = 45, }; enum sd_log_level_t { diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index 9ed2875ad..73b98937e 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -3407,7 +3407,6 @@ class Linear : public UnaryBlock { bool bias; bool force_f32; bool force_prec_f32; - bool allow_weight_scale; bool has_weight_scale = false; bool int8_convrot = false; int int8_convrot_group_size = 0; @@ -3430,8 +3429,11 @@ class Linear : public UnaryBlock { } auto weight_storage = tensor_storage_map.find(prefix + "weight"); const bool is_int8_tensorwise = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise; - if ((allow_weight_scale || is_int8_tensorwise) && tensor_storage_map.find(prefix + "weight_scale") != tensor_storage_map.end()) { - params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features); + auto weight_scale_storage = tensor_storage_map.find(prefix + "weight_scale"); + if (weight_scale_storage != tensor_storage_map.end()) { + const int64_t scale_nelements = weight_scale_storage->second.nelements(); + GGML_ASSERT(scale_nelements == 1 || scale_nelements == out_features); + params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements); has_weight_scale = true; } if (is_int8_tensorwise) { @@ -3445,17 +3447,15 @@ class Linear : public UnaryBlock { public: Linear(int64_t in_features, int64_t out_features, - bool bias = true, - bool force_f32 = false, - bool force_prec_f32 = false, - float scale = 1.f, - bool allow_weight_scale = false) + bool bias = true, + bool force_f32 = false, + bool force_prec_f32 = false, + float scale = 1.f) : in_features(in_features), out_features(out_features), bias(bias), force_f32(force_f32), force_prec_f32(force_prec_f32), - allow_weight_scale(allow_weight_scale), scale(scale) {} void set_scale(float scale_) { @@ -3467,7 +3467,11 @@ class Linear : public UnaryBlock { } ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { - ggml_tensor* w = params["weight"]; + ggml_tensor* w = params["weight"]; + ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr; + if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) { + w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16); + } ggml_tensor* b = nullptr; if (bias) { b = params["bias"]; @@ -3498,7 +3502,7 @@ class Linear : public UnaryBlock { out = ggml_ext_linear_i8_tensorwise(ctx->ggml_ctx, x, w, - params["weight_scale"], + weight_scale, b, int8_convrot ? int8_convrot_group_size : 0, scale); @@ -3517,6 +3521,30 @@ class Linear : public UnaryBlock { } return out; } + if (has_weight_scale) { + out = ggml_ext_linear(ctx->ggml_ctx, x, w, nullptr, force_prec_f32, scale); + out = ggml_mul(ctx->ggml_ctx, out, weight_scale); + if (ctx->weight_adapter) { + WeightAdapter::ForwardParams forward_params; + forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR; + forward_params.linear.force_prec_f32 = force_prec_f32; + forward_params.linear.scale = scale; + out = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx, + ctx->backend, + x, + w, + out, + prefix, + forward_params); + if (b != nullptr) { + b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias"); + } + } + if (b != nullptr) { + out = ggml_add_inplace(ctx->ggml_ctx, out, b); + } + return out; + } if (ctx->weight_adapter) { WeightAdapter::ForwardParams forward_params; forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR; @@ -3526,12 +3554,6 @@ class Linear : public UnaryBlock { } else { out = ggml_ext_linear(ctx->ggml_ctx, x, w, linear_bias, force_prec_f32, scale); } - if (has_weight_scale) { - out = ggml_mul(ctx->ggml_ctx, out, params["weight_scale"]); - if (b != nullptr) { - out = ggml_add_inplace(ctx->ggml_ctx, out, b); - } - } return out; } }; diff --git a/src/model/diffusion/ideogram4.hpp b/src/model/diffusion/ideogram4.hpp index bfa2f86a4..6ce8e1fab 100644 --- a/src/model/diffusion/ideogram4.hpp +++ b/src/model/diffusion/ideogram4.hpp @@ -142,7 +142,7 @@ namespace Ideogram4 { __STATIC_INLINE__ std::shared_ptr make_linear(int64_t in_features, int64_t out_features, bool bias = true) { - return std::make_shared(in_features, out_features, bias, false, false, 1.f, true); + return std::make_shared(in_features, out_features, bias); } __STATIC_INLINE__ std::vector gen_ideogram4_pe(int grid_h, diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp index 69bcaa1ec..1d1269488 100644 --- a/src/model_io/safetensors_io.cpp +++ b/src/model_io/safetensors_io.cpp @@ -87,9 +87,9 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) { } else if (dtype == "F64") { ttype = GGML_TYPE_F32; } else if (dtype == "F8_E4M3") { - ttype = GGML_TYPE_F16; + ttype = GGML_TYPE_F8_E4M3; } else if (dtype == "F8_E5M2") { - ttype = GGML_TYPE_F16; + ttype = GGML_TYPE_F8_E5M2; } else if (dtype == "I32") { ttype = GGML_TYPE_I32; } else if (dtype == "I64") { @@ -328,12 +328,10 @@ bool read_safetensors_file(const std::string& file_path, bool tensor_size_ok; if (dtype == "F8_E4M3") { tensor_storage.is_f8_e4m3 = true; - // f8 -> f16 - tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2); + tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size); } else if (dtype == "F8_E5M2") { tensor_storage.is_f8_e5m2 = true; - // f8 -> f16 - tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2); + tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size); } else if (dtype == "F64") { tensor_storage.is_f64 = true; // f64 -> f32 diff --git a/src/model_io/tensor_storage.h b/src/model_io/tensor_storage.h index 5672c9437..11f53e063 100644 --- a/src/model_io/tensor_storage.h +++ b/src/model_io/tensor_storage.h @@ -54,9 +54,7 @@ struct TensorStorage { } int64_t nbytes_to_read() const { - if (is_f8_e4m3 || is_f8_e5m2) { - return nbytes() / 2; - } else if (is_f64 || is_i64) { + if (is_f64 || is_i64) { return nbytes() * 2; } else { return nbytes(); diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 891a41736..dc2bb4bf0 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -78,66 +78,6 @@ bool is_unused_tensor(const std::string& name) { return false; } -uint16_t f8_e4m3_to_f16(uint8_t f8) { - // do we need to support uz? - - const uint32_t exponent_bias = 7; - if (f8 == 0xff) { - return ggml_fp32_to_fp16(-NAN); - } else if (f8 == 0x7f) { - return ggml_fp32_to_fp16(NAN); - } - - uint32_t sign = f8 & 0x80; - uint32_t exponent = (f8 & 0x78) >> 3; - uint32_t mantissa = f8 & 0x07; - uint32_t result = sign << 24; - if (exponent == 0) { - if (mantissa > 0) { - exponent = 0x7f - exponent_bias; - - // yes, 2 times - if ((mantissa & 0x04) == 0) { - mantissa &= 0x03; - mantissa <<= 1; - exponent -= 1; - } - if ((mantissa & 0x04) == 0) { - mantissa &= 0x03; - mantissa <<= 1; - exponent -= 1; - } - - result |= (mantissa & 0x03) << 21; - result |= exponent << 23; - } - } else { - result |= mantissa << 20; - exponent += 0x7f - exponent_bias; - result |= exponent << 23; - } - - return ggml_fp32_to_fp16(*reinterpret_cast(&result)); -} - -uint16_t f8_e5m2_to_f16(uint8_t fp8) { - return static_cast(fp8) << 8; -} - -void f8_e4m3_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) { - // support inplace op - for (int64_t i = n - 1; i >= 0; i--) { - dst[i] = f8_e4m3_to_f16(src[i]); - } -} - -void f8_e5m2_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) { - // support inplace op - for (int64_t i = n - 1; i >= 0; i--) { - dst[i] = f8_e5m2_to_f16(src[i]); - } -} - void f64_to_f32_vec(double* src, float* dst, int64_t n) { // support inplace op for (int64_t i = 0; i < n; i++) { @@ -929,9 +869,7 @@ std::vector ModelLoader::mmap_tensors(std::maptype) { continue; @@ -1215,11 +1153,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, read_time_ms.fetch_add(t1 - t0); t0 = ggml_time_ms(); - if (tensor_storage.is_f8_e4m3) { - f8_e4m3_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements()); - } else if (tensor_storage.is_f8_e5m2) { - f8_e5m2_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements()); - } else if (tensor_storage.is_f64) { + if (tensor_storage.is_f64) { f64_to_f32_vec((double*)read_buf, (float*)target_buf, tensor_storage.nelements()); } else if (tensor_storage.is_i64) { i64_to_i32_vec((int64_t*)read_buf, (int32_t*)target_buf, tensor_storage.nelements()); diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index 126a4ddbc..1e73d42b3 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -1569,6 +1569,11 @@ std::string convert_tensor_name(std::string name, SDVersion version) { } } + static const std::vector> generic_name_map = { + {".scale_weight", ".weight_scale"}, + }; + replace_with_name_map(name, generic_name_map); + if (is_lora) { name = "lora." + name; }