Add VoxCPM v1 — lightweight VoxCPM TTS support (0.5B)

CMakeLists.txt

@@ -762,6 +762,23 @@ audiocpp_add_model(voxcpm2
         engine::models::voxcpm2::make_voxcpm2_loader
 )
 
+audiocpp_add_model(voxcpm1
+    SOURCES
+        src/community_models/voxcpm1/assets.cpp
+        src/community_models/voxcpm1/audiovae.cpp
+        src/community_models/voxcpm1/config_gguf.cpp
+        src/community_models/voxcpm1/generator.cpp
+        src/community_models/voxcpm1/gguf_metadata.cpp
+        src/community_models/voxcpm1/minicpm.cpp
+        src/community_models/voxcpm1/session.cpp
+        src/community_models/voxcpm1/tokenizer_gguf.cpp
+        src/community_models/voxcpm1/tokenizer_text.cpp
+    INCLUDES
+        engine/community_models/voxcpm1/session.h
+    LOADERS
+        engine::community_models::voxcpm1::make_voxcpm1_loader
+)
+
 audiocpp_add_model(vibevoice
     SOURCES
         src/models/vibevoice/assets.cpp

docs/tts.md

@@ -15,6 +15,7 @@
 | NeuTTS | `neutts` | `tts` | [NeuTTS](#neutts) |
 | OmniVoice | `omnivoice` | `tts` | [OmniVoice](#omnivoice), [full guide](models/omnivoice.md) |
 | PocketTTS | `pocket_tts` | `tts` | [PocketTTS](#pockettts) |
+| VoxCPM1 | `voxcpm1` | `tts` | [VoxCPM1](#voxcpm1) |
 | VoxCPM2 | `voxcpm2` | `tts`, `vdes` | [VoxCPM2](#voxcpm2) |
 | Higgs Audio v3 TTS | `higgs_audio_tts` | `tts` | [Higgs Audio v3 TTS](#higgs-audio-v3-tts) |
 | Fish Audio S2 Pro | `fish_audio` | `tts` | [Fish Audio S2 Pro](#fish-audio-s2-pro) |
@@ -422,6 +423,49 @@ audiocpp_cli --task tts --family pocket_tts --model models/pocket-tts --backend
 | `--text-chunk-size` | integer chars | `256` | Long-form chunk size. |
 | `--session-option pocket_tts.voice_state_cache_slots=<n>` | integer slots | `4` | Prepared voice-state cache slots; set `0` to disable reuse. |
 
+## VoxCPM1
+
+VoxCPM1 supports offline and streaming TTS plus short-reference voice cloning. It reuses the VoxCPM2 runtime tree with a GGUF tensor-adaptation layer that understands the OpenBMB folded AudioVAE weights. The registered package is the 16 kHz 0.5B model; the runtime is size-agnostic, so a different VoxCPM1 GGUF can still be loaded via an explicit `--model <path>`.
+
+| Field | Value |
+|---|---|
+| Family | `voxcpm1` |
+| Model directory | `models/VoxCPM1-GGUF` (0.5B) |
+| Task | `tts` |
+| Modes | `offline`, `streaming` |
+| Languages | Model auto-handles supported languages |
+| Voice input | Optional reference WAV; optional transcript through `--reference-text` |
+| Built-in voices | Not exposed |
+
+Text to speech:
+
+```bash
+audiocpp_cli --task tts --family voxcpm1 --model models/VoxCPM1-GGUF/voxcpm-0.5b-q8_0-audiovae-f16.gguf --backend cpu --text "Hello from VoxCPM1." --out out.wav
+```
+
+Voice clone:
+
+```bash
+audiocpp_cli --task tts --family voxcpm1 --model models/VoxCPM1-GGUF/voxcpm-0.5b-q8_0-audiovae-f16.gguf --backend cpu --text "Hello from VoxCPM1." --voice-ref assets/resources/b.wav --out out.wav
+```
+
+Streaming output:
+
+```bash
+audiocpp_cli --task tts --family voxcpm1 --model models/VoxCPM1-GGUF/voxcpm-0.5b-q8_0-audiovae-f16.gguf --backend cpu --mode streaming --text "Hello from VoxCPM1." --request-option retry_badcase=false --out out.wav
+```
+
+| Option | Values | Default | Meaning |
+|---|---:|---:|---|
+| `--voice-ref` | WAV path | not set | Reference speaker audio. |
+| `--reference-text` | text | empty string | Transcript for the reference audio (clone prompting). |
+| `--mode` | `offline`, `streaming` | `offline` | Full-output or streaming run mode; streaming requires `retry_badcase=false`. |
+| `--session-option voxcpm1.mem_saver=true\|false` | bool | `false` | Use tighter graph workspaces and release MiniCPM/AudioVAE request graphs after completion to reduce resident VRAM. |
+| `--session-option voxcpm1.prompt_cache_slots=<n>` | integer | `1` | Prompt and prompt-audio embedding cache slots. Set to `0` to disable prompt caching. |
+| `--max-tokens` | integer | `4096` | Maximum generated AR tokens. |
+| `--num-inference-steps` | integer | `10` | Flow matching steps. |
+| `--guidance-scale` | float | `2.0` | CFG strength. |
+
 ## VoxCPM2
 
 VoxCPM2 supports plain TTS, voice design, controllable voice cloning, and an ultimate-clone style that uses both prompt audio and transcript. The CLI expresses voice design with the same text convention as the upstream examples: put the voice/style description in parentheses at the start of `--text`.

include/engine/community_models/voxcpm1/assets.h

@@ -0,0 +1,102 @@
+#pragma once
+
+#include "engine/framework/assets/resource_bundle.h"
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/community_models/voxcpm1/tokenizer_gguf.h"
+
+#include <cstdint>
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace engine::community_models::voxcpm1 {
+
+struct VoxCPM1RopeScalingConfig {
+    std::string type;
+    std::vector<float> long_factor;
+    std::vector<float> short_factor;
+    int64_t original_max_position_embeddings = 0;
+};
+
+struct VoxCPM1MiniCPMConfig {
+    int64_t bos_token_id = 1;
+    int64_t eos_token_id = 2;
+    int64_t hidden_size = 0;
+    int64_t intermediate_size = 0;
+    int64_t max_position_embeddings = 0;
+    int64_t num_attention_heads = 0;
+    int64_t num_hidden_layers = 0;
+    int64_t num_key_value_heads = 0;
+    int64_t kv_channels = 0;
+    int64_t vocab_size = 0;
+    int64_t scale_emb = 1;
+    int64_t dim_model_base = 0;
+    float rms_norm_eps = 1.0e-5F;
+    float rope_theta = 10000.0F;
+    float scale_depth = 1.0F;
+    bool use_mup = false;
+    bool no_rope = false;
+    VoxCPM1RopeScalingConfig rope_scaling;
+};
+
+struct VoxCPM1LocalTransformerConfig {
+    int64_t hidden_dim = 0;
+    int64_t ffn_dim = 0;
+    int64_t num_heads = 0;
+    int64_t num_layers = 0;
+    int64_t kv_channels = 0;
+};
+
+struct VoxCPM1CFMConfig {
+    float sigma_min = 1.0e-6F;
+    std::string solver = "euler";
+    std::string t_scheduler = "log-norm";
+    float inference_cfg_rate = 2.0F;
+};
+
+struct VoxCPM1DiTConfig : VoxCPM1LocalTransformerConfig {
+    bool mean_mode = false;
+    VoxCPM1CFMConfig cfm;
+};
+
+struct VoxCPM1AudioVAEConfig {
+    int64_t encoder_dim = 0;
+    std::vector<int64_t> encoder_rates;
+    int64_t latent_dim = 0;
+    int64_t decoder_dim = 0;
+    std::vector<int64_t> decoder_rates;
+    std::vector<int64_t> sample_rate_bin_boundaries;
+    int sample_rate = 0;
+    int output_sample_rate = 0;
+};
+
+struct VoxCPM1Config {
+    std::string architecture;
+    VoxCPM1MiniCPMConfig lm;
+    int64_t patch_size = 4;
+    int64_t feat_dim = 64;
+    int64_t residual_lm_num_layers = 8;
+    bool residual_lm_no_rope = false;
+    int64_t scalar_quantization_latent_dim = 512;
+    int64_t scalar_quantization_scale = 9;
+    VoxCPM1LocalTransformerConfig encoder;
+    VoxCPM1DiTConfig dit;
+    VoxCPM1AudioVAEConfig audio_vae;
+    int64_t max_length = 8192;
+    std::string device = "cuda";
+    std::string dtype = "bfloat16";
+    bool v1 = false;
+};
+
+struct VoxCPM1Assets {
+    assets::ResourceBundle resources;
+    VoxCPM1Config config;
+    std::shared_ptr<const assets::TensorSource> model_weights;
+    std::shared_ptr<const assets::TensorSource> audiovae_weights;
+    std::shared_ptr<const VoxCPM1GgufTokenizer> gguf_tokenizer;
+};
+
+std::shared_ptr<const VoxCPM1Assets> load_voxcpm1_assets(const std::filesystem::path & model_path);
+
+}  // namespace engine::community_models::voxcpm1

include/engine/community_models/voxcpm1/audiovae.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include "engine/framework/core/backend.h"
+#include "engine/framework/runtime/session.h"
+#include "engine/community_models/voxcpm1/assets.h"
+#include "engine/community_models/voxcpm1/types.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace engine::core {
+class ExecutionContext;
+}
+
+namespace engine::community_models::voxcpm1 {
+
+struct VoxCPM1AudioVAEDecoderConfig {
+  size_t weight_context_bytes = 768ull * 1024ull * 1024ull;
+  size_t graph_context_bytes = 1024ull * 1024ull * 1024ull;
+  size_t encoder_graph_context_bytes = 1024ull * 1024ull * 1024ull;
+  int64_t latent_frame_capacity = 0;
+  int64_t encoder_sample_capacity = 240000;
+  engine::assets::TensorStorageType weight_storage_type =
+      engine::assets::TensorStorageType::F32;
+};
+
+class VoxCPM1AudioVAEDecoderRuntime final {
+public:
+  VoxCPM1AudioVAEDecoderRuntime(
+      std::shared_ptr<const VoxCPM1Assets> assets,
+      engine::core::ExecutionContext &execution_context,
+      VoxCPM1AudioVAEDecoderConfig config = {});
+  ~VoxCPM1AudioVAEDecoderRuntime();
+
+  runtime::AudioBuffer decode_features(const std::vector<float> &features,
+                                       int64_t patches);
+  VoxCPM1EncodedPrompt encode_prompt_audio(
+      const std::optional<runtime::AudioBuffer> &prompt_audio,
+      const std::string &prompt_text,
+      const std::optional<runtime::AudioBuffer> &reference_audio);
+  void release_runtime_memory();
+  void release_encoder_graph();
+
+private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+} // namespace engine::community_models::voxcpm1

include/engine/community_models/voxcpm1/config_gguf.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "engine/community_models/voxcpm1/assets.h"
+#include "engine/framework/assets/tensor_source.h"
+
+#include <memory>
+#include <optional>
+#include <string>
+
+namespace engine::community_models::voxcpm1 {
+
+// Load VoxCPM1 config from GGUF metadata
+VoxCPM1Config load_voxcpm1_config_from_gguf(const engine::assets::TensorSource & source);
+
+// Check if GGUF has VoxCPM1 config metadata
+bool has_voxcpm1_config_metadata(const engine::assets::TensorSource & source);
+
+}  // namespace engine::community_models::voxcpm1

include/engine/community_models/voxcpm1/generator.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/community_models/voxcpm1/types.h"
+
+#include <cstddef>
+#include <functional>
+#include <memory>
+#include <string>
+
+namespace engine::core {
+class ExecutionContext;
+}
+
+namespace engine::community_models::voxcpm1 {
+
+struct VoxCPM1Assets;
+
+struct VoxCPM1FeatureGeneratorConfig {
+  size_t weight_context_bytes = 3ull * 1024ull * 1024ull * 1024ull;
+  size_t text_embedding_graph_context_bytes = 64ull * 1024ull * 1024ull;
+  size_t lm_step_graph_context_bytes = 1024ull * 1024ull * 1024ull;
+  size_t projection_graph_context_bytes = 256ull * 1024ull * 1024ull;
+  size_t local_encoder_graph_context_bytes = 512ull * 1024ull * 1024ull;
+  size_t dit_graph_context_bytes = 1024ull * 1024ull * 1024ull;
+  size_t prompt_cache_slots = 1;
+  bool mem_saver = false;
+  engine::assets::TensorStorageType weight_storage_type =
+      engine::assets::TensorStorageType::Native;
+};
+
+class VoxCPM1FeatureGeneratorRuntime final {
+public:
+  VoxCPM1FeatureGeneratorRuntime(
+      std::shared_ptr<const VoxCPM1Assets> assets,
+      engine::core::ExecutionContext &execution_context,
+      VoxCPM1FeatureGeneratorConfig config = {});
+  ~VoxCPM1FeatureGeneratorRuntime();
+
+  VoxCPM1Result generate_zero_shot(const std::string &text,
+                                   const VoxCPM1GenerationOptions &options);
+  VoxCPM1Result generate(const std::string &text,
+                         const VoxCPM1EncodedPrompt *prompt,
+                         const VoxCPM1GenerationOptions &options);
+  VoxCPM1StreamingResult
+  generate_streaming(const std::string &text,
+                     const VoxCPM1EncodedPrompt *prompt,
+                     const VoxCPM1GenerationOptions &options,
+                     const std::function<void(const VoxCPM1StreamingChunk &)>
+                         &chunk_callback = nullptr);
+  void release_runtime_memory();
+  void release_text_length_memory();
+
+private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+} // namespace engine::community_models::voxcpm1

include/engine/community_models/voxcpm1/gguf_metadata.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+struct gguf_context;
+
+namespace engine::assets {
+class TensorSource;
+}
+
+namespace engine::community_models::voxcpm1 {
+
+// Reads GGUF KV metadata (tokenizer.ggml.*, voxcpm_*) directly from the file
+// backing a TensorSource. Only meaningful for GGUF sources: for any other
+// source type valid() is false and all accessors return nullopt (optional_*)
+// or throw (require_*). This keeps VoxCPM schema knowledge out of the
+// framework TensorSource interface.
+class GgufMetadataReader {
+public:
+    explicit GgufMetadataReader(const engine::assets::TensorSource & source);
+    ~GgufMetadataReader();
+
+    GgufMetadataReader(const GgufMetadataReader &) = delete;
+    GgufMetadataReader & operator=(const GgufMetadataReader &) = delete;
+
+    bool valid() const noexcept { return gguf_ != nullptr; }
+
+    [[nodiscard]] std::optional<std::string> optional_string(std::string_view key) const;
+    [[nodiscard]] std::optional<uint32_t> optional_u32(std::string_view key) const;
+    [[nodiscard]] std::optional<float> optional_f32(std::string_view key) const;
+    [[nodiscard]] std::optional<std::vector<std::string>> optional_string_array(std::string_view key) const;
+    [[nodiscard]] std::optional<std::vector<int32_t>> optional_i32_array(std::string_view key) const;
+    [[nodiscard]] std::optional<std::vector<float>> optional_f32_array(std::string_view key) const;
+
+    [[nodiscard]] std::string require_string(std::string_view key) const;
+    [[nodiscard]] uint32_t require_u32(std::string_view key) const;
+    [[nodiscard]] std::vector<std::string> require_string_array(std::string_view key) const;
+    [[nodiscard]] std::vector<int32_t> require_i32_array(std::string_view key) const;
+
+private:
+    struct gguf_context * gguf_ = nullptr;
+};
+
+}  // namespace engine::community_models::voxcpm1