diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index e04b2b0c..276c65d5 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -11,6 +11,7 @@ #include "engine/framework/debug/trace.h" #include "engine/framework/io/json.h" #include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/errors.h" #include "engine/framework/runtime/registry.h" #include @@ -1001,6 +1002,12 @@ HttpResponse ServerState::handle(const HttpRequest & request) { else { response = error_response(404, "unknown endpoint: " + request.path, "not_found"); } + } catch (const engine::runtime::CapacityError & ex) { + // The request is too big for the device, which is the caller's to fix -- + // reporting it as 500 sends them looking for a server fault that is not + // there. Checked before ServerBusyError only because both are + // runtime_error; the two conditions are disjoint. + response = error_response(400, ex.what(), "invalid_request_error"); } catch (const ServerBusyError & ex) { // Non-streaming requests surface the busy state as 503 before any response is // sent. (Streaming requests acquire the lock inside the stream body, after diff --git a/include/engine/framework/runtime/errors.h b/include/engine/framework/runtime/errors.h new file mode 100644 index 00000000..fe1c3c12 --- /dev/null +++ b/include/engine/framework/runtime/errors.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +namespace engine::runtime { + +// A request the device cannot serve AT THIS SIZE -- e.g. a transcription +// prompt plus audio whose prefill graph does not fit in VRAM. Distinct from a +// genuine internal fault: the caller can fix it by sending less, so servers +// should surface it as a client error rather than an opaque 500. +class CapacityError : public std::runtime_error { +public: + explicit CapacityError(const std::string & message) : std::runtime_error(message) {} +}; + +} // namespace engine::runtime diff --git a/src/models/qwen3_asr/thinker.cpp b/src/models/qwen3_asr/thinker.cpp index ac710686..e039cd4a 100644 --- a/src/models/qwen3_asr/thinker.cpp +++ b/src/models/qwen3_asr/thinker.cpp @@ -12,6 +12,7 @@ #include "engine/framework/modules/positional_modules.h" #include "engine/framework/modules/primitive_modules.h" #include "engine/framework/modules/structural_modules.h" +#include "engine/framework/runtime/errors.h" #include "engine/framework/runtime/kv_cache.h" #include "engine/framework/sampling/decode_modules.h" @@ -334,7 +335,14 @@ class PrefillGraph { ggml_build_forward_expand(graph_, logits_); buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), runtime_->backend()); if (buffer_ == nullptr) { - throw std::runtime_error("failed to allocate Qwen3 ASR thinker prefill graph"); + // Size, not a fault: the graph scales with prompt_steps_, which the + // caller controls through the transcription prompt and the length of + // the audio. Say which, and by how much, so the remedy is obvious. + throw engine::runtime::CapacityError( + "Qwen3 ASR prefill graph does not fit in device memory at this size (" + + std::to_string(prompt_steps_) + " prompt steps, of which " + + std::to_string(audio_tokens_) + " are audio tokens); " + "shorten the transcription prompt or the audio"); } const auto pos = modules::qwen_position_ids(prompt_steps_); ggml_backend_tensor_set(positions_, pos.data(), 0, pos.size() * sizeof(int32_t)); diff --git a/src/models/voxcpm2/generator.cpp b/src/models/voxcpm2/generator.cpp index aeffe256..3186a142 100644 --- a/src/models/voxcpm2/generator.cpp +++ b/src/models/voxcpm2/generator.cpp @@ -11,6 +11,7 @@ #include "engine/framework/modules/structural_modules.h" #include "engine/framework/modules/weight_binding.h" #include "engine/framework/runtime/cache_slots.h" +#include "engine/framework/runtime/errors.h" #include "engine/framework/sampling/torch_random.h" #include "engine/models/voxcpm2/assets.h" #include "engine/models/voxcpm2/minicpm.h" @@ -1373,7 +1374,12 @@ class VoxCPM2FeatureGeneratorRuntime::Impl { if (sequence.rows.empty() || static_cast(sequence.rows.size()) >= config.max_length) { - throw std::runtime_error("VoxCPM2 prompt exceeds model cache length"); + // Caller-controlled: the prompt audio/text decides how many rows this + // is. Report the numbers so the remedy is arithmetic, not guesswork. + throw engine::runtime::CapacityError( + "VoxCPM2 prompt exceeds the model cache length (" + + std::to_string(sequence.rows.size()) + " rows, limit " + + std::to_string(config.max_length) + "); shorten the prompt"); } return sequence; } @@ -1487,7 +1493,13 @@ class VoxCPM2FeatureGeneratorRuntime::Impl { const int64_t patch_elems = config.patch_size * config.feat_dim; if (static_cast(prefill.rows.size()) + max_tokens > config.max_length) { - throw std::runtime_error("VoxCPM2 generation exceeds model cache length"); + // Same: prefill rows come from the input text, max_tokens from the + // request. Both are the caller's to reduce. + throw engine::runtime::CapacityError( + "VoxCPM2 generation exceeds the model cache length (" + + std::to_string(prefill.rows.size()) + " prefill rows + " + + std::to_string(max_tokens) + " requested tokens, limit " + + std::to_string(config.max_length) + "); shorten the input text"); } base_lm_.reset(); residual_lm_.reset();