zipvoice: add k2-fsa ZipVoice community TTS with zh/en frontend

CMakeLists.txt

@@ -479,6 +479,8 @@ add_library(engine_core OBJECT
     src/framework/decoders/tdt_decoder_greedy_separate_heads.cpp
     src/framework/modules/linear_module.cpp
     src/framework/modules/packed_linear_weights.cpp
+    src/framework/modules/convnext_modules.cpp
+    src/framework/modules/vocoders/vocos_vocoder.cpp
     src/framework/modules/primitive_modules.cpp
     src/framework/modules/activation_modules.cpp
     src/framework/modules/norm_modules.cpp
@@ -827,6 +829,21 @@ audiocpp_add_model(f5_tts
         habibi_tts
 )
 
+audiocpp_add_model(zipvoice
+    SOURCES
+        src/community_models/zipvoice/session.cpp
+        src/community_models/zipvoice/synthesize.cpp
+        src/community_models/zipvoice/emilia_tokenizer.cpp
+        src/community_models/zipvoice/jieba_segmenter.cpp
+        src/community_models/zipvoice/weights.cpp
+        src/community_models/zipvoice/zipformer.cpp
+    INCLUDES
+        engine/community_models/zipvoice/session.h
+        engine/community_models/zipvoice/synthesize.h
+    LOADERS
+        engine::models::zipvoice::make_zipvoice_loader
+)
+
 audiocpp_add_model(minimax_h3
     SOURCES
         src/community_models/minimax_h3/assets.cpp
@@ -2901,6 +2918,16 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
             endif()
         endif()
         # Model-specific tests and probes.
+        if (zipvoice IN_LIST AUDIOCPP_LINKED_MODELS)
+            add_executable(zipvoice_parity tests/zipvoice/zipvoice_parity_main.cpp)
+            target_link_libraries(zipvoice_parity PRIVATE engine_runtime ggml)
+            target_include_directories(zipvoice_parity PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+            if (ENGINE_ENABLE_OPENMP)
+                target_link_libraries(zipvoice_parity PRIVATE OpenMP::OpenMP_CXX)
+            endif()
+            add_executable(zipvoice_zh_tokens tests/zipvoice/zipvoice_zh_tokens_main.cpp)
+            target_link_libraries(zipvoice_zh_tokens PRIVATE engine_runtime)
+        endif()
         if (f5_tts IN_LIST AUDIOCPP_LINKED_MODELS)
             target_compile_definitions(engine_model_f5_tts PRIVATE F5_MEL_TEST=1)
             foreach(f5_test IN ITEMS f5_e2e f5_parity f5_cfg_parity f5_tokenizer)

README.md

@@ -172,6 +172,7 @@ Community model ports live under `community_models` to make the ownership bounda
 | **echo_tts** | Clone | en | GGUF 16/Q8 | [@5uck1ess](https://github.com/5uck1ess) | [Echo-TTS](docs/community_models/echo_tts.md) 44.1 kHz zero-shot voice cloning with EchoDiT latents and Fish S1-DAC decoding |
 | **f5_tts** | TTS, Clone | en, ar (Habibi) | GGUF | [@tareko](https://github.com/tareko) | [F5-TTS](docs/community_models/f5_tts.md) flow-matching DiT synthesis and voice cloning, with Habibi Arabic aliases `habibi`/`habibi_tts` |
 | **glm_tts** | TTS, Clone | zh, en | GGUF | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](docs/community_models/glm_tts.md) zero-shot synthesis and voice cloning support |
+| **zipvoice** | TTS, Clone | zh, en | GGUF F32 | Community | [ZipVoice](docs/community_models/zipvoice.md) k2-fsa TTSZipformer flow-matching zero-shot voice cloning with Vocos vocoder; jieba + pypinyin Chinese frontend |
 | **granite5asr** | ASR | en | GGUF Q8 | [@ampersandru](https://github.com/ampersandru) | [IBM Granite Speech 5.0 470M TurboCTC](docs/community_models/granite5asr.md) ultra-fast Conformer-CTC ASR with Shaw relative positional embeddings and ByteLevel BPE |
 | **inflect_v2** | TTS | en | GGUF FP32 | Jan [@JanWerder](https://github.com/JanWerder) | [Inflect Micro v2 and Nano v2](docs/community_models/inflect_v2.md) native offline synthesis |
 | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Safetensors, GGUF Q8 | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](docs/community_models/kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps |

docs/community_models/models.md

@@ -21,6 +21,7 @@ Practical expectations:
 | **echo_tts** | TTS, voice cloning | en | Tym [@5uck1ess](https://github.com/5uck1ess), [@dignome](https://github.com/dignome) | [Echo-TTS](echo_tts.md) 44.1 kHz zero-shot voice cloning: 2.8B diffusion transformer in 80-D PCA space, decoded by the Fish S1-DAC autoencoder. Byte-level text, no phonemiser, no reference transcript |
 | **f5_tts** | TTS, voice cloning | en, ar (Habibi) | Community | [F5-TTS](f5_tts.md) flow-matching DiT — M0 scaffolding, aliases `habibi`/`habibi_tts` |
 | **glm_tts** | TTS, voice cloning | zh, en | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](glm_tts.md) zero-shot synthesis and voice cloning support |
+| **zipvoice** | TTS, voice cloning | zh, en | Community | [ZipVoice](zipvoice.md) k2-fsa TTSZipformer flow-matching zero-shot cloning with Vocos vocoder; distilled 8-step and base variants, hosted GGUF package |
 | **granite5asr** | ASR | en | Community | [IBM Granite Speech 5.0 470M TurboCTC](granite5asr.md) ultra-fast Conformer-CTC ASR with Shaw relative positional embeddings and ByteLevel BPE |
 | **inflect_v2** | TTS | en | Community | [Inflect Micro v2 and Nano v2](inflect_v2.md) native FP32 offline synthesis |
 | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps |

docs/community_models/zipvoice.md

@@ -0,0 +1,169 @@
+# ZipVoice (community model)
+
+[ZipVoice](https://github.com/k2-fsa/ZipVoice) (k2-fsa) is a zero-shot voice-cloning TTS built on a
+TTSZipformer flow-matching backbone with a Vocos mel-24kHz vocoder. `zipvoice_distill` is the
+distilled variant (8 Euler steps, guidance-scale embedding); the base model uses batched
+classifier-free guidance. Both share one architecture and one GGUF packaging.
+
+**Status: inference complete.** Text encoder, duration/ratio conditioning, flow-matching decoder,
+and the Vocos vocoder are verified against the reference PyTorch implementation with golden
+fixtures (`tests/zipvoice/zipvoice_parity_main.cpp`): per-layer and per-submodule encoder taps,
+text conditioning, sampled features, single- and batched velocity fields (odd/even/full lengths,
+both timesteps), fbank, and vocos audio — all at cosine 1.0. The Chinese/English text frontend
+(`emilia` mode) is verified token-for-token against the upstream `EmiliaTokenizer`
+(`tests/zipvoice/zipvoice_zh_tokens_main.cpp`).
+
+## Highlights
+
+- Zero-shot voice cloning from a short reference clip + transcript (zh and en)
+- Duration prediction from the prompt speaking rate (`speed` option scales it)
+- Distill model: 8 Euler steps with guidance-scale embedding; base model: batched CFG
+- Chinese + English mixed text via the `emilia` frontend (jieba + pypinyin tables + espeak-ng),
+  inline pinyin overrides (`<zhong1> <guo2>`)
+- Ready-made GGUF package hosted at [davidxifeng/zipvoice-gguf](https://huggingface.co/davidxifeng/zipvoice-gguf) (Apache-2.0 upstream); local re-conversion with the tools below also works
+
+## Conversion
+
+A ready-made, self-contained package is hosted at
+[davidxifeng/zipvoice-gguf](https://huggingface.co/davidxifeng/zipvoice-gguf) — the model
+manager downloads it directly (`zipvoice-distill-orig.gguf`, flow-matching model + bundled
+Vocos + embedded frontend sidecars). The Q8_0 package includes the same resources:
+
+```bash
+python3 tools/model_manager_v2.py install zipvoice_distill_q8_0
+```
+
+To rebuild it locally:
+
+```bash
+# 1. stage the Chinese frontend tables (requires the upstream ZipVoice python env
+#    for pypinyin; writes zh_chars.tsv / zh_phrases.tsv / zh_syllables.tsv and
+#    downloads the pinned jieba dictionaries next to tokens.txt)
+python3 tools/community_models/export_zipvoice_zh_dict.py \
+    --output-dir /models/ZipVoice/zipvoice_distill
+
+# 2. flatten + package (torch checkpoint -> safetensors -> GGUF with model.* and vocos.*)
+python3 tools/community_models/convert_zipvoice.py \
+    --model-dir /models/ZipVoice/zipvoice_distill \
+    --vocos /models/vocos-mel-24khz/vocos.safetensors \
+    --converter build/bin/audiocpp_gguf
+```
+
+The converter needs `tokens.txt` + `model.json` + `model.pt` in `--model-dir` (the HF
+`k2-fsa/ZipVoice` `zipvoice_distill` snapshot layout). When the directory was staged by
+`export_zipvoice_zh_dict.py`, the `zh_*` frontend sidecars are embedded into the GGUF alongside
+`tokens.txt` / `model.json`, so a converted package is a single self-sufficient file (loose
+copies are still staged next to it for the directory layout). With `--safetensors-only` it
+stops at the development format (`zipvoice-orig.safetensors` + config + vocab in one
+directory) — an intermediate for tooling and the direct synthesis API; the CLI loads GGUF
+packages.
+
+## CLI usage
+
+```bash
+# English cloning (espeak frontend, the default)
+audiocpp_cli --task clon --family zipvoice \
+    --model /models/ZipVoice-Distill-GGUF/zipvoice-distill-orig.gguf \
+    --session-option zipvoice.espeak_library_path=/opt/homebrew/lib/libespeak-ng.dylib \
+    --voice-ref prompt.wav --reference-text "Reference transcript." \
+    --text "Text to synthesize." --out out.wav
+
+# Chinese / mixed text (emilia frontend, built in; the zh_* tables are embedded in the
+# GGUF, or staged by export_zipvoice_zh_dict.py next to it)
+audiocpp_cli --task clon --family zipvoice \
+    --model ... \
+    --session-option zipvoice.espeak_library_path=/opt/homebrew/lib/libespeak-ng.dylib \
+    --voice-ref prompt.wav --reference-text "参考文本。" \
+    --text "要合成的文本。" --out out.wav
+```
+
+The text frontend is fixed to the EmiliaTokenizer pipeline (zh/en/mixed; the upstream
+default): `tokenizer` is no longer a session option. Session options: `zipvoice.vocos_path`
+(only for safetensors checkpoints without a bundled vocoder), `zipvoice.espeak_library_path`,
+`zipvoice.espeak_data_path`, `zipvoice.num_inference_steps`, `zipvoice.guidance_scale`,
+`zipvoice.t_shift`. Requests accept `reference_text` (required), `guidance_scale`,
+`num_inference_steps`, `t_shift`, `speed`, `feat_scale`, `target_rms`, `seed`, `lang` (espeak
+voice for English segments), and `token_ids`/`prompt_token_ids` to bypass the frontend entirely
+(direct API callers can also still select the espeak/simple frontends through
+`ZipVoiceSynthesisRequest::tokenizer`).
+
+## Frontend details (emilia mode)
+
+The upstream default `EmiliaTokenizer` pipeline is reproduced exactly:
+Chinese text normalization (framework `ChineseTextNormalizer`) → punctuation mapping → jieba
+segmentation (a model-local port of the Jieba maximum-probability DAG and four-state BMES HMM,
+`src/community_models/zipvoice/jieba_segmenter.cpp`, same dictionaries as python jieba) → pypinyin
+readings from baked tables (TONE3 syllables split into initial+`0` / final+tone tokens) → tone sandhi
+(3rd-tone runs, 一/不) applied per jieba word → token mapping with OOV skipping; English runs are
+phonemized with espeak-ng. Verification compares C++ token ids against the upstream tokenizer
+run in the ZipVoice python environment (`tests/zipvoice/zh_reference_ids.py`).
+
+The jieba segmentation port is derived from [cppjieba](https://github.com/yanyiwu/cppjieba)
+(Copyright (c) 2013 Yanyi Wu) and [jieba](https://github.com/fxsjy/jieba) (Copyright (c) 2012 Sun
+Junyi), both MIT licensed; the `zh_jieba_dict.txt` / `zh_hmm_model.txt` resources are the jieba
+dictionaries and are embedded in the GGUF.
+
+## Performance
+
+The measured ZipVoice-Distill ggml Metal runs achieve **RTF < 1**, the target
+for community models. RTF is synthesis wall time divided by generated audio
+duration; lower is better.
+
+Measured on **Apple M3**, **Metal** backend, on **2026-09-18**, with a
+Release build and 8 CPU threads, using
+`zipvoice-distill-orig.gguf`, 8 Euler steps, guidance scale 3, time shift 0.5,
+speed 1, seed 666, and the 24 kHz Vocos vocoder. Each reference pair was tested
+with one discarded warmup followed by three timed calls in the same process.
+Wall time includes tokenization, reference preprocessing, feature extraction,
+text encoding, the solver, and Vocos. Model loading and initial compilation
+are excluded by warmup; reference-file reading and output WAV writing are
+outside the timer.
+
+| Reference audio / transcript | Generated audio | Mean synthesis wall time | Mean RTF | Meets RTF < 1 |
+|---|---:|---:|---:|---|
+| `zh-male-ref.wav` / `zh-male-ref.txt` | 19.115 s | 4.697 s | **0.2457** | Yes |
+| `zh-ref.wav` / `zh-ref-text.txt` | 12.843 s | 2.332 s | **0.1816** | Yes |
+
+The individual measured RTFs were `0.245406`, `0.245910`, `0.245799` for
+the male reference and `0.181118`, `0.181592`, `0.182100` for the female
+reference. Both used this exact target text, including the space in “宁 静”:
+
+> 风声渐息，落叶不再沙沙作响。小狐狸缓缓合上双眼，在平缓绵长的呼吸声中，安然步入宁 静的梦境。
+
+The measured implementation includes the session-owned runtime, framework Vocos graph, and
+model-local Jieba implementation. It measures the direct synthesis API with
+one persistent `ZipVoiceComputeDevice` per reference pair; session-level
+long-text chunking is not exercised. The two benchmarks ran sequentially.
+The discarded first calls took 6.744 s (male) and 2.772 s (female), including
+lazy model loading and graph/kernel setup but excluding resource-bundle
+resolution and input-file reading.
+
+Both cases meet the community-model target. Results cover these two inputs
+on one machine; CPU and CUDA performance are not established by this measurement.
+Local benchmark source, full logs, and output WAVs are saved in
+`outputs/zipvoice-metal-current-20260918/` (not distributed with the model).
+
+## Parity
+
+Select `--backend metal` for Apple GPU inference or `--backend cpu` for the
+CPU reference path. The text encoder, flow decoder and Vocos backbone run on
+the selected backend; feature extraction and the final ISTFT run on the host.
+The direct C++ API defaults to `BestAvailable`, while sessions honor the requested
+backend, including an explicit CPU selection.
+
+CPU/Metal parity is tested on Apple M3 with the Distill GGUF, including odd
+sequence lengths, batched velocity evaluation, Vocos and eight-step synthesis.
+ZipVoice requests F32 matrix products to avoid FP16 staging error accumulating
+through the sampler. CUDA uses the same graphs but has not been verified on
+hardware in this GPU validation run.
+
+```bash
+# golden fixtures: tests/zipvoice/build_reference.py (upstream env) + parity harness
+build/bin/zipvoice_parity reference.npz <model-path> reference.npz vocos.safetensors out.wav
+build/bin/zipvoice_zh_tokens zh_reference_ids.json <model-dir>
+```
+
+Weights and runtime graphs are owned by the `ZipVoiceComputeDevice` runtime (one
+per session), so callers injecting a borrowed backend must release the device
+runtime (for example with `zipvoice_clear_runtime(device)`) before destroying
+that backend, after all ZipVoice calls using it have finished.

docs/espeak_phonemizer.md

@@ -5,7 +5,11 @@ eSpeak-ng. SanoTTS (E2M and Piper frontends) and Inflect v2 use it. Other models
 including the separate Kokoro preview, can use the same adapter without copying
 dynamic-library loading or process-global state management.
 
-By default users provide an installed shared library and its matching data.
+By default users provide an installed shared library and its matching data. With no explicit
+session paths, the adapter first tries common install locations (Homebrew
+`/opt/homebrew/lib` and `/usr/local/lib` on macOS, multiarch lib directories on Linux) by
+soname, and eSpeak then falls back to its build-time data path; pass
+`espeak_library_path` / `espeak_data_path` when the install lives elsewhere.
 Alternatively, `AUDIOCPP_STATIC_ESPEAK=ON` builds the pinned eSpeak-ng 1.52.0 source
 and statically links its code into both CLI and server. No eSpeak DLL or `.so` is
 required in that mode. Existing explicit library/data session options still work.

include/engine/community_models/zipvoice/emilia_tokenizer.h

@@ -0,0 +1,80 @@
+#pragma once
+
+// Chinese/English mixed text frontend for ZipVoice, mirroring the upstream
+// EmiliaTokenizer (zipvoice/tokenizer/tokenizer.py). Chinese segments are
+// converted with baked pypinyin tables (tone3 syllables split into
+// initial+"0" / final+tone tokens, e.g. "w0 o3") plus run-time tone sandhi;
+// English segments are phonemized with eSpeak-ng; <pin3yin1> overrides use
+// the bare-syllable table. Output token ids follow tokens.txt with OOV
+// tokens skipped, exactly like the reference.
+
+#include <cstdint>
+#include <filesystem>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace engine::audio {
+class EspeakPhonemizer;
+}
+
+namespace engine::models::zipvoice {
+
+class JiebaSegmenter;
+
+class EmiliaTokenizer {
+public:
+    struct EspeakConfig {
+        std::string library_path;  // optional: libespeak-ng path
+        std::string data_path;     // optional: espeak-ng data directory
+        std::string lang = "en-us";
+    };
+
+    // Frontend table locations. The spec-registered resource bundle hands
+    // over materialized GGUF-embedded sidecars; development directories and
+    // direct API callers fill the paths from the model directory.
+    struct TablePaths {
+        std::filesystem::path chars;       // zh_chars.tsv
+        std::filesystem::path phrases;     // zh_phrases.tsv
+        std::filesystem::path syllables;   // zh_syllables.tsv
+        std::filesystem::path jieba_dict;  // zh_jieba_dict.txt
+        std::filesystem::path hmm_model;   // zh_hmm_model.txt
+
+        // Loose-file layout: everything in one model directory.
+        static TablePaths from_model_dir(const std::filesystem::path & dir) {
+            return {dir / "zh_chars.tsv", dir / "zh_phrases.tsv",
+                    dir / "zh_syllables.tsv", dir / "zh_jieba_dict.txt",
+                    dir / "zh_hmm_model.txt"};
+        }
+    };
+
+    // `vocab` is the tokens.txt token -> id map; OOV phones are skipped.
+    // The Chinese tables are optional only if no Chinese text is encoded.
+    EmiliaTokenizer(const TablePaths & tables,
+                    const std::unordered_map<std::string, int32_t> & vocab,
+                    const EspeakConfig & espeak);
+    ~EmiliaTokenizer();  // out-of-line: owns an incomplete-type unique_ptr
+
+    std::vector<int32_t> encode(const std::string & text) const;
+
+private:
+    std::unordered_map<std::string, int32_t> vocab_;
+    TablePaths tables_;
+    std::unordered_map<std::string, std::vector<std::string>> chars_;
+    std::unordered_map<std::string, std::vector<std::string>> phrases_;
+    std::unordered_map<std::string, std::vector<std::string>> syllables_;
+    size_t max_phrase_codepoints_ = 0;
+    EspeakConfig espeak_;
+    // Lazily created so a pure-Chinese workload never touches eSpeak.
+    mutable std::unique_ptr<audio::EspeakPhonemizer> phonemizer_;
+    // Lazily created jieba segmenter (model-local MixSegment port = python
+    // jieba.cut with HMM): word boundaries define reading lookup and the
+    // scope of tone-sandhi application, mirroring the reference pipeline
+    // lazy_pinyin(jieba.cut(text), tone_sandhi=True).
+    mutable std::unique_ptr<JiebaSegmenter> segmenter_;
+    mutable std::mutex segmenter_mutex_;
+};
+
+}  // namespace engine::models::zipvoice

include/engine/community_models/zipvoice/session.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "engine/framework/runtime/model.h"
+
+#include <memory>
+#include <string>
+
+namespace engine::models::zipvoice {
+
+std::shared_ptr<runtime::IVoiceModelLoader> make_zipvoice_loader();
+
+}  // namespace engine::models::zipvoice