diff --git a/CMakeLists.txt b/CMakeLists.txt index 3fa2768fe..e28d392ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,8 @@ add_library(engine_core OBJECT src/framework/decoders/tdt_decoder_greedy_separate_heads.cpp src/framework/modules/linear_module.cpp src/framework/modules/packed_linear_weights.cpp + src/framework/modules/convnext_modules.cpp + src/framework/modules/vocoders/vocos_vocoder.cpp src/framework/modules/primitive_modules.cpp src/framework/modules/activation_modules.cpp src/framework/modules/norm_modules.cpp @@ -827,6 +829,21 @@ audiocpp_add_model(f5_tts habibi_tts ) +audiocpp_add_model(zipvoice + SOURCES + src/community_models/zipvoice/session.cpp + src/community_models/zipvoice/synthesize.cpp + src/community_models/zipvoice/emilia_tokenizer.cpp + src/community_models/zipvoice/jieba_segmenter.cpp + src/community_models/zipvoice/weights.cpp + src/community_models/zipvoice/zipformer.cpp + INCLUDES + engine/community_models/zipvoice/session.h + engine/community_models/zipvoice/synthesize.h + LOADERS + engine::models::zipvoice::make_zipvoice_loader +) + audiocpp_add_model(minimax_h3 SOURCES src/community_models/minimax_h3/assets.cpp @@ -2901,6 +2918,16 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST endif() endif() # Model-specific tests and probes. + if (zipvoice IN_LIST AUDIOCPP_LINKED_MODELS) + add_executable(zipvoice_parity tests/zipvoice/zipvoice_parity_main.cpp) + target_link_libraries(zipvoice_parity PRIVATE engine_runtime ggml) + target_include_directories(zipvoice_parity PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + if (ENGINE_ENABLE_OPENMP) + target_link_libraries(zipvoice_parity PRIVATE OpenMP::OpenMP_CXX) + endif() + add_executable(zipvoice_zh_tokens tests/zipvoice/zipvoice_zh_tokens_main.cpp) + target_link_libraries(zipvoice_zh_tokens PRIVATE engine_runtime) + endif() if (f5_tts IN_LIST AUDIOCPP_LINKED_MODELS) target_compile_definitions(engine_model_f5_tts PRIVATE F5_MEL_TEST=1) foreach(f5_test IN ITEMS f5_e2e f5_parity f5_cfg_parity f5_tokenizer) diff --git a/README.md b/README.md index cfc46696b..ac7f5f482 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ Community model ports live under `community_models` to make the ownership bounda | **echo_tts** | Clone | en | GGUF 16/Q8 | [@5uck1ess](https://github.com/5uck1ess) | [Echo-TTS](docs/community_models/echo_tts.md) 44.1 kHz zero-shot voice cloning with EchoDiT latents and Fish S1-DAC decoding | | **f5_tts** | TTS, Clone | en, ar (Habibi) | GGUF | [@tareko](https://github.com/tareko) | [F5-TTS](docs/community_models/f5_tts.md) flow-matching DiT synthesis and voice cloning, with Habibi Arabic aliases `habibi`/`habibi_tts` | | **glm_tts** | TTS, Clone | zh, en | GGUF | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](docs/community_models/glm_tts.md) zero-shot synthesis and voice cloning support | +| **zipvoice** | TTS, Clone | zh, en | GGUF F32 | Community | [ZipVoice](docs/community_models/zipvoice.md) k2-fsa TTSZipformer flow-matching zero-shot voice cloning with Vocos vocoder; jieba + pypinyin Chinese frontend | | **granite5asr** | ASR | en | GGUF Q8 | [@ampersandru](https://github.com/ampersandru) | [IBM Granite Speech 5.0 470M TurboCTC](docs/community_models/granite5asr.md) ultra-fast Conformer-CTC ASR with Shaw relative positional embeddings and ByteLevel BPE | | **inflect_v2** | TTS | en | GGUF FP32 | Jan [@JanWerder](https://github.com/JanWerder) | [Inflect Micro v2 and Nano v2](docs/community_models/inflect_v2.md) native offline synthesis | | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Safetensors, GGUF Q8 | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](docs/community_models/kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps | diff --git a/docs/community_models/models.md b/docs/community_models/models.md index cc4553412..41c5a0cd5 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -21,6 +21,7 @@ Practical expectations: | **echo_tts** | TTS, voice cloning | en | Tym [@5uck1ess](https://github.com/5uck1ess), [@dignome](https://github.com/dignome) | [Echo-TTS](echo_tts.md) 44.1 kHz zero-shot voice cloning: 2.8B diffusion transformer in 80-D PCA space, decoded by the Fish S1-DAC autoencoder. Byte-level text, no phonemiser, no reference transcript | | **f5_tts** | TTS, voice cloning | en, ar (Habibi) | Community | [F5-TTS](f5_tts.md) flow-matching DiT — M0 scaffolding, aliases `habibi`/`habibi_tts` | | **glm_tts** | TTS, voice cloning | zh, en | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](glm_tts.md) zero-shot synthesis and voice cloning support | +| **zipvoice** | TTS, voice cloning | zh, en | Community | [ZipVoice](zipvoice.md) k2-fsa TTSZipformer flow-matching zero-shot cloning with Vocos vocoder; distilled 8-step and base variants, hosted GGUF package | | **granite5asr** | ASR | en | Community | [IBM Granite Speech 5.0 470M TurboCTC](granite5asr.md) ultra-fast Conformer-CTC ASR with Shaw relative positional embeddings and ByteLevel BPE | | **inflect_v2** | TTS | en | Community | [Inflect Micro v2 and Nano v2](inflect_v2.md) native FP32 offline synthesis | | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps | diff --git a/docs/community_models/zipvoice.md b/docs/community_models/zipvoice.md new file mode 100644 index 000000000..1411d2b6c --- /dev/null +++ b/docs/community_models/zipvoice.md @@ -0,0 +1,169 @@ +# ZipVoice (community model) + +[ZipVoice](https://github.com/k2-fsa/ZipVoice) (k2-fsa) is a zero-shot voice-cloning TTS built on a +TTSZipformer flow-matching backbone with a Vocos mel-24kHz vocoder. `zipvoice_distill` is the +distilled variant (8 Euler steps, guidance-scale embedding); the base model uses batched +classifier-free guidance. Both share one architecture and one GGUF packaging. + +**Status: inference complete.** Text encoder, duration/ratio conditioning, flow-matching decoder, +and the Vocos vocoder are verified against the reference PyTorch implementation with golden +fixtures (`tests/zipvoice/zipvoice_parity_main.cpp`): per-layer and per-submodule encoder taps, +text conditioning, sampled features, single- and batched velocity fields (odd/even/full lengths, +both timesteps), fbank, and vocos audio — all at cosine 1.0. The Chinese/English text frontend +(`emilia` mode) is verified token-for-token against the upstream `EmiliaTokenizer` +(`tests/zipvoice/zipvoice_zh_tokens_main.cpp`). + +## Highlights + +- Zero-shot voice cloning from a short reference clip + transcript (zh and en) +- Duration prediction from the prompt speaking rate (`speed` option scales it) +- Distill model: 8 Euler steps with guidance-scale embedding; base model: batched CFG +- Chinese + English mixed text via the `emilia` frontend (jieba + pypinyin tables + espeak-ng), + inline pinyin overrides (` `) +- Ready-made GGUF package hosted at [davidxifeng/zipvoice-gguf](https://huggingface.co/davidxifeng/zipvoice-gguf) (Apache-2.0 upstream); local re-conversion with the tools below also works + +## Conversion + +A ready-made, self-contained package is hosted at +[davidxifeng/zipvoice-gguf](https://huggingface.co/davidxifeng/zipvoice-gguf) — the model +manager downloads it directly (`zipvoice-distill-orig.gguf`, flow-matching model + bundled +Vocos + embedded frontend sidecars). The Q8_0 package includes the same resources: + +```bash +python3 tools/model_manager_v2.py install zipvoice_distill_q8_0 +``` + +To rebuild it locally: + +```bash +# 1. stage the Chinese frontend tables (requires the upstream ZipVoice python env +# for pypinyin; writes zh_chars.tsv / zh_phrases.tsv / zh_syllables.tsv and +# downloads the pinned jieba dictionaries next to tokens.txt) +python3 tools/community_models/export_zipvoice_zh_dict.py \ + --output-dir /models/ZipVoice/zipvoice_distill + +# 2. flatten + package (torch checkpoint -> safetensors -> GGUF with model.* and vocos.*) +python3 tools/community_models/convert_zipvoice.py \ + --model-dir /models/ZipVoice/zipvoice_distill \ + --vocos /models/vocos-mel-24khz/vocos.safetensors \ + --converter build/bin/audiocpp_gguf +``` + +The converter needs `tokens.txt` + `model.json` + `model.pt` in `--model-dir` (the HF +`k2-fsa/ZipVoice` `zipvoice_distill` snapshot layout). When the directory was staged by +`export_zipvoice_zh_dict.py`, the `zh_*` frontend sidecars are embedded into the GGUF alongside +`tokens.txt` / `model.json`, so a converted package is a single self-sufficient file (loose +copies are still staged next to it for the directory layout). With `--safetensors-only` it +stops at the development format (`zipvoice-orig.safetensors` + config + vocab in one +directory) — an intermediate for tooling and the direct synthesis API; the CLI loads GGUF +packages. + +## CLI usage + +```bash +# English cloning (espeak frontend, the default) +audiocpp_cli --task clon --family zipvoice \ + --model /models/ZipVoice-Distill-GGUF/zipvoice-distill-orig.gguf \ + --session-option zipvoice.espeak_library_path=/opt/homebrew/lib/libespeak-ng.dylib \ + --voice-ref prompt.wav --reference-text "Reference transcript." \ + --text "Text to synthesize." --out out.wav + +# Chinese / mixed text (emilia frontend, built in; the zh_* tables are embedded in the +# GGUF, or staged by export_zipvoice_zh_dict.py next to it) +audiocpp_cli --task clon --family zipvoice \ + --model ... \ + --session-option zipvoice.espeak_library_path=/opt/homebrew/lib/libespeak-ng.dylib \ + --voice-ref prompt.wav --reference-text "参考文本。" \ + --text "要合成的文本。" --out out.wav +``` + +The text frontend is fixed to the EmiliaTokenizer pipeline (zh/en/mixed; the upstream +default): `tokenizer` is no longer a session option. Session options: `zipvoice.vocos_path` +(only for safetensors checkpoints without a bundled vocoder), `zipvoice.espeak_library_path`, +`zipvoice.espeak_data_path`, `zipvoice.num_inference_steps`, `zipvoice.guidance_scale`, +`zipvoice.t_shift`. Requests accept `reference_text` (required), `guidance_scale`, +`num_inference_steps`, `t_shift`, `speed`, `feat_scale`, `target_rms`, `seed`, `lang` (espeak +voice for English segments), and `token_ids`/`prompt_token_ids` to bypass the frontend entirely +(direct API callers can also still select the espeak/simple frontends through +`ZipVoiceSynthesisRequest::tokenizer`). + +## Frontend details (emilia mode) + +The upstream default `EmiliaTokenizer` pipeline is reproduced exactly: +Chinese text normalization (framework `ChineseTextNormalizer`) → punctuation mapping → jieba +segmentation (a model-local port of the Jieba maximum-probability DAG and four-state BMES HMM, +`src/community_models/zipvoice/jieba_segmenter.cpp`, same dictionaries as python jieba) → pypinyin +readings from baked tables (TONE3 syllables split into initial+`0` / final+tone tokens) → tone sandhi +(3rd-tone runs, 一/不) applied per jieba word → token mapping with OOV skipping; English runs are +phonemized with espeak-ng. Verification compares C++ token ids against the upstream tokenizer +run in the ZipVoice python environment (`tests/zipvoice/zh_reference_ids.py`). + +The jieba segmentation port is derived from [cppjieba](https://github.com/yanyiwu/cppjieba) +(Copyright (c) 2013 Yanyi Wu) and [jieba](https://github.com/fxsjy/jieba) (Copyright (c) 2012 Sun +Junyi), both MIT licensed; the `zh_jieba_dict.txt` / `zh_hmm_model.txt` resources are the jieba +dictionaries and are embedded in the GGUF. + +## Performance + +The measured ZipVoice-Distill ggml Metal runs achieve **RTF < 1**, the target +for community models. RTF is synthesis wall time divided by generated audio +duration; lower is better. + +Measured on **Apple M3**, **Metal** backend, on **2026-09-18**, with a +Release build and 8 CPU threads, using +`zipvoice-distill-orig.gguf`, 8 Euler steps, guidance scale 3, time shift 0.5, +speed 1, seed 666, and the 24 kHz Vocos vocoder. Each reference pair was tested +with one discarded warmup followed by three timed calls in the same process. +Wall time includes tokenization, reference preprocessing, feature extraction, +text encoding, the solver, and Vocos. Model loading and initial compilation +are excluded by warmup; reference-file reading and output WAV writing are +outside the timer. + +| Reference audio / transcript | Generated audio | Mean synthesis wall time | Mean RTF | Meets RTF < 1 | +|---|---:|---:|---:|---| +| `zh-male-ref.wav` / `zh-male-ref.txt` | 19.115 s | 4.697 s | **0.2457** | Yes | +| `zh-ref.wav` / `zh-ref-text.txt` | 12.843 s | 2.332 s | **0.1816** | Yes | + +The individual measured RTFs were `0.245406`, `0.245910`, `0.245799` for +the male reference and `0.181118`, `0.181592`, `0.182100` for the female +reference. Both used this exact target text, including the space in “宁 静”: + +> 风声渐息，落叶不再沙沙作响。小狐狸缓缓合上双眼，在平缓绵长的呼吸声中，安然步入宁 静的梦境。 + +The measured implementation includes the session-owned runtime, framework Vocos graph, and +model-local Jieba implementation. It measures the direct synthesis API with +one persistent `ZipVoiceComputeDevice` per reference pair; session-level +long-text chunking is not exercised. The two benchmarks ran sequentially. +The discarded first calls took 6.744 s (male) and 2.772 s (female), including +lazy model loading and graph/kernel setup but excluding resource-bundle +resolution and input-file reading. + +Both cases meet the community-model target. Results cover these two inputs +on one machine; CPU and CUDA performance are not established by this measurement. +Local benchmark source, full logs, and output WAVs are saved in +`outputs/zipvoice-metal-current-20260918/` (not distributed with the model). + +## Parity + +Select `--backend metal` for Apple GPU inference or `--backend cpu` for the +CPU reference path. The text encoder, flow decoder and Vocos backbone run on +the selected backend; feature extraction and the final ISTFT run on the host. +The direct C++ API defaults to `BestAvailable`, while sessions honor the requested +backend, including an explicit CPU selection. + +CPU/Metal parity is tested on Apple M3 with the Distill GGUF, including odd +sequence lengths, batched velocity evaluation, Vocos and eight-step synthesis. +ZipVoice requests F32 matrix products to avoid FP16 staging error accumulating +through the sampler. CUDA uses the same graphs but has not been verified on +hardware in this GPU validation run. + +```bash +# golden fixtures: tests/zipvoice/build_reference.py (upstream env) + parity harness +build/bin/zipvoice_parity reference.npz reference.npz vocos.safetensors out.wav +build/bin/zipvoice_zh_tokens zh_reference_ids.json +``` + +Weights and runtime graphs are owned by the `ZipVoiceComputeDevice` runtime (one +per session), so callers injecting a borrowed backend must release the device +runtime (for example with `zipvoice_clear_runtime(device)`) before destroying +that backend, after all ZipVoice calls using it have finished. diff --git a/docs/espeak_phonemizer.md b/docs/espeak_phonemizer.md index a48ab5f30..62094fabd 100644 --- a/docs/espeak_phonemizer.md +++ b/docs/espeak_phonemizer.md @@ -5,7 +5,11 @@ eSpeak-ng. SanoTTS (E2M and Piper frontends) and Inflect v2 use it. Other models including the separate Kokoro preview, can use the same adapter without copying dynamic-library loading or process-global state management. -By default users provide an installed shared library and its matching data. +By default users provide an installed shared library and its matching data. With no explicit +session paths, the adapter first tries common install locations (Homebrew +`/opt/homebrew/lib` and `/usr/local/lib` on macOS, multiarch lib directories on Linux) by +soname, and eSpeak then falls back to its build-time data path; pass +`espeak_library_path` / `espeak_data_path` when the install lives elsewhere. Alternatively, `AUDIOCPP_STATIC_ESPEAK=ON` builds the pinned eSpeak-ng 1.52.0 source and statically links its code into both CLI and server. No eSpeak DLL or `.so` is required in that mode. Existing explicit library/data session options still work. diff --git a/include/engine/community_models/zipvoice/emilia_tokenizer.h b/include/engine/community_models/zipvoice/emilia_tokenizer.h new file mode 100644 index 000000000..f4def0006 --- /dev/null +++ b/include/engine/community_models/zipvoice/emilia_tokenizer.h @@ -0,0 +1,80 @@ +#pragma once + +// Chinese/English mixed text frontend for ZipVoice, mirroring the upstream +// EmiliaTokenizer (zipvoice/tokenizer/tokenizer.py). Chinese segments are +// converted with baked pypinyin tables (tone3 syllables split into +// initial+"0" / final+tone tokens, e.g. "w0 o3") plus run-time tone sandhi; +// English segments are phonemized with eSpeak-ng; overrides use +// the bare-syllable table. Output token ids follow tokens.txt with OOV +// tokens skipped, exactly like the reference. + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::audio { +class EspeakPhonemizer; +} + +namespace engine::models::zipvoice { + +class JiebaSegmenter; + +class EmiliaTokenizer { +public: + struct EspeakConfig { + std::string library_path; // optional: libespeak-ng path + std::string data_path; // optional: espeak-ng data directory + std::string lang = "en-us"; + }; + + // Frontend table locations. The spec-registered resource bundle hands + // over materialized GGUF-embedded sidecars; development directories and + // direct API callers fill the paths from the model directory. + struct TablePaths { + std::filesystem::path chars; // zh_chars.tsv + std::filesystem::path phrases; // zh_phrases.tsv + std::filesystem::path syllables; // zh_syllables.tsv + std::filesystem::path jieba_dict; // zh_jieba_dict.txt + std::filesystem::path hmm_model; // zh_hmm_model.txt + + // Loose-file layout: everything in one model directory. + static TablePaths from_model_dir(const std::filesystem::path & dir) { + return {dir / "zh_chars.tsv", dir / "zh_phrases.tsv", + dir / "zh_syllables.tsv", dir / "zh_jieba_dict.txt", + dir / "zh_hmm_model.txt"}; + } + }; + + // `vocab` is the tokens.txt token -> id map; OOV phones are skipped. + // The Chinese tables are optional only if no Chinese text is encoded. + EmiliaTokenizer(const TablePaths & tables, + const std::unordered_map & vocab, + const EspeakConfig & espeak); + ~EmiliaTokenizer(); // out-of-line: owns an incomplete-type unique_ptr + + std::vector encode(const std::string & text) const; + +private: + std::unordered_map vocab_; + TablePaths tables_; + std::unordered_map> chars_; + std::unordered_map> phrases_; + std::unordered_map> syllables_; + size_t max_phrase_codepoints_ = 0; + EspeakConfig espeak_; + // Lazily created so a pure-Chinese workload never touches eSpeak. + mutable std::unique_ptr phonemizer_; + // Lazily created jieba segmenter (model-local MixSegment port = python + // jieba.cut with HMM): word boundaries define reading lookup and the + // scope of tone-sandhi application, mirroring the reference pipeline + // lazy_pinyin(jieba.cut(text), tone_sandhi=True). + mutable std::unique_ptr segmenter_; + mutable std::mutex segmenter_mutex_; +}; + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/session.h b/include/engine/community_models/zipvoice/session.h new file mode 100644 index 000000000..9e46b733e --- /dev/null +++ b/include/engine/community_models/zipvoice/session.h @@ -0,0 +1,12 @@ +#pragma once + +#include "engine/framework/runtime/model.h" + +#include +#include + +namespace engine::models::zipvoice { + +std::shared_ptr make_zipvoice_loader(); + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/synthesize.h b/include/engine/community_models/zipvoice/synthesize.h new file mode 100644 index 000000000..869f82766 --- /dev/null +++ b/include/engine/community_models/zipvoice/synthesize.h @@ -0,0 +1,151 @@ +#pragma once + +#include "engine/framework/core/backend.h" + +#include +#include +#include +#include + +namespace engine::assets { +class ResourceBundle; +} + +namespace engine::models::zipvoice { + +class ZipVoiceRuntimeState; +std::shared_ptr make_zipvoice_runtime_state(); + +// Compute device for the whole zipvoice stack (text encoder, flow-matching +// decoder, Vocos backbone). GPU first: the default BestAvailable resolves to +// Metal/CUDA when a usable accelerator exists and falls back to CPU +// otherwise; `threads` applies to CPU compute. A runtime that already owns a +// backend may inject it via `backend`. Destroy/reset this device runtime before +// freeing a borrowed backend. Copies share runtime ownership and serialize use. +struct ZipVoiceComputeDevice { + core::BackendType backend_type = core::BackendType::BestAvailable; + int device_index = 0; + int threads = 0; // 0 = hardware concurrency + ggml_backend_t backend = nullptr; + std::shared_ptr runtime = make_zipvoice_runtime_state(); +}; + +// Release this session/device's weights and graphs; no process-global cache. +void zipvoice_clear_runtime(const ZipVoiceComputeDevice & device); + +struct ZipVoiceSynthesisRequest { + std::string text; + std::string ref_text; + std::vector ref_audio; + int ref_sample_rate = 24000; + int ref_channels = 1; // ref_audio is interleaved + std::vector token_ids; // optional: pre-tokenized target text + std::vector prompt_token_ids; // optional: pre-tokenized ref text + std::string tokenizer = "emilia"; // fixed frontend: zh/en/mixed (espeak for en runs) + std::string lang = "en-us"; + std::string espeak_library_path; + std::string espeak_data_path; + int num_steps = 8; + float guidance_scale = 3.0F; + float t_shift = 0.5F; + float speed = 1.0F; + float feat_scale = 0.1F; + float target_rms = 0.1F; + uint32_t seed = 666; + bool fixed_seed = true; +}; + +struct ZipVoiceSynthesisResult { + std::vector audio; // 24 kHz mono + int sample_rate = 24000; + double model_seconds = 0.0; // wall time of the flow + vocoder + double audio_seconds = 0.0; // generated audio duration +}; + +// Full pipeline: tokenize (if needed) -> fbank -> duration prediction -> +// Euler flow-matching sampling -> Vocos decode. `model_path` is the GGUF +// package or the safetensors development directory. `resources` (optional) +// is the spec-resolved bundle: registered sidecars (tokens, model_config, +// zh_* tables, materialized from an embedded-sidecar GGUF or found in the +// development directory) take priority over loose files next to the +// checkpoint; direct API callers (parity harnesses) may pass nullptr. +ZipVoiceSynthesisResult zipvoice_synthesize( + const std::string & model_path, + const std::string & vocos_path, + const ZipVoiceSynthesisRequest & request, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// --- test hooks (parity harness) ------------------------------------------- + +// Backend Vocos path using the same cached model/backend as synthesis. +std::vector zipvoice_vocos_decode_on_device( + const std::string & model_path, + const std::string & vocos_path, + const std::vector & mel_rows, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources = nullptr); + +// Text-condition stage: token ids -> text_condition [T, feat_dim] given +// prompt feature length. Mirrors forward_text_inference_ratio_duration. +std::vector zipvoice_text_condition( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + int64_t prompt_features_len, + float speed, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// Raw text-encoder output for pre-tokenized ids (pad token appended +// internally, mirroring pad_labels): returns [S+1, feat_dim] row-major. +// `layer_taps` (optional) receives per-encoder-layer outputs as [S+1, C] +// rows for parity bisecting. +struct ZipVoiceLayerTaps { + std::vector> layers; // per-layer outputs [S+1, C] + std::vector> stages; // first-layer submodule taps +}; +std::vector zipvoice_text_encoder_raw( + const std::string & model_path, + const std::vector & token_ids, + ZipVoiceLayerTaps * layer_taps = nullptr, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// One flow-matching velocity evaluation at time t (single batch, no CFG). +std::vector zipvoice_velocity( + const std::string & model_path, + const std::vector & xt, // [T, feat_dim] + const std::vector & text_condition, + const std::vector & speech_condition, + int64_t features_len, + float t, + float guidance_scale, + const ZipVoiceComputeDevice & device = {}, + int batch_size = 1, + const engine::assets::ResourceBundle * resources = nullptr); + +// Full sampler: x0 -> x1 (mirrors ZipVoice::sample for duration="predict"). +std::vector zipvoice_sample( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + const std::vector & prompt_features, // [T_prompt, feat_dim], scaled + int64_t prompt_features_len, + const std::vector & x0, + int num_steps, + float guidance_scale, + float t_shift, + float speed, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// Log-mel filterbank identical to VocosFbank (24 kHz, 100 mels, hop 256, +// power=1, log clamp 1e-7, htk scale), with lhotse frame-count alignment. +std::vector zipvoice_logmel(const std::vector & wav); + +// Vocos mel-24kHz decode (same weights as f5_tts uses). +std::vector zipvoice_vocos_decode( + const std::string & vocos_path, const std::vector & mel_frames); + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/weights.h b/include/engine/community_models/zipvoice/weights.h new file mode 100644 index 000000000..dc56a5730 --- /dev/null +++ b/include/engine/community_models/zipvoice/weights.h @@ -0,0 +1,111 @@ +#pragma once + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/module.h" + +#include +#include +#include +#include + +namespace engine::models::zipvoice { + +// Architecture hyper-parameters, mirroring model.json "model" + tokenizer info. +struct ZipVoiceConfig { + std::vector fm_downsampling_factor = {1, 2, 4, 2, 1}; + std::vector fm_num_layers = {2, 2, 4, 4, 4}; + std::vector fm_cnn_kernel = {31, 15, 7, 15, 31}; + int fm_feedforward_dim = 1536; + int fm_num_heads = 4; + int fm_dim = 512; + int text_num_layers = 4; + int text_feedforward_dim = 512; + int text_cnn_kernel = 9; + int text_num_heads = 4; + int text_dim = 192; + int time_embed_dim = 192; + int text_embed_dim = 192; + int query_head_dim = 32; + int value_head_dim = 12; + int pos_head_dim = 4; + int pos_dim = 48; + int feat_dim = 100; + int vocab_size = 360; // from tokens.txt + int pad_id = 0; + int sampling_rate = 24000; + bool guidance_scale_embed = false; // ZipVoice-Distill +}; + +// One Zipformer2 encoder layer's weights (torch module names). +struct ZipLayerWeights { + core::TensorValue bypass_scale; // [C] + core::TensorValue bypass_mid_scale; // [C] + core::TensorValue attn_in_proj_w; // [C, (2*qh+ph)*H] + core::TensorValue attn_in_proj_b; + core::TensorValue linear_pos_w; // [H*ph, pos_dim] + core::TensorValue sa1_in_w, sa1_in_b; // [H*vh, C] + core::TensorValue sa1_out_w, sa1_out_b; // [C, H*vh] + core::TensorValue sa2_in_w, sa2_in_b; + core::TensorValue sa2_out_w, sa2_out_b; + core::TensorValue ff1_in_w, ff1_in_b; + core::TensorValue ff1_out_w, ff1_out_b; + core::TensorValue ff2_in_w, ff2_in_b; + core::TensorValue ff2_out_w, ff2_out_b; + core::TensorValue ff3_in_w, ff3_in_b; + core::TensorValue ff3_out_w, ff3_out_b; + core::TensorValue na_in_w, na_in_b; // [3*hidden, C] + core::TensorValue na_out_w, na_out_b; // [C, hidden] + core::TensorValue cm1_in_w, cm1_in_b; + core::TensorValue cm1_conv_w, cm1_conv_b; // [C, 1, k], [C] + core::TensorValue cm1_out_w, cm1_out_b; + core::TensorValue cm2_in_w, cm2_in_b; + core::TensorValue cm2_conv_w, cm2_conv_b; + core::TensorValue cm2_out_w, cm2_out_b; + core::TensorValue norm_bias; // [C] + float norm_log_scale = 0.0F; +}; + +struct ZipStackWeights { + std::vector layers; + // present when the stack has a time embedding projection + core::TensorValue time_proj_w, time_proj_b; // [C, time_dim] + // present when downsampling_factor > 1 + std::vector downsample_bias; // [ds] (softmax applied at load) + core::TensorValue out_combiner_scale; // [C] +}; + +struct TTSZipformerWeights { + core::TensorValue in_proj_w, in_proj_b; + core::TensorValue out_proj_w, out_proj_b; + core::TensorValue embed_w; // only for text encoder: [vocab, text_embed_dim] + // fm decoder only: + core::TensorValue time_mlp0_w, time_mlp0_b; // [2*tdim, tdim] + core::TensorValue time_mlp2_w, time_mlp2_b; // [tdim, 2*tdim] + core::TensorValue guidance_embed_w; // [tdim, tdim] no bias (distill) + std::vector stacks; +}; + +struct ZipVoiceWeights { + std::shared_ptr store; + TTSZipformerWeights fm_decoder; + TTSZipformerWeights text_encoder; +}; + +// Loads a converted checkpoint. `prefix` is "model" for GGUF packages and "" +// for the development safetensors export. Throws on shape mismatch. +// `config_path` (optional) overrides `model_dir / "model.json"` for +// spec-registered (materialized) sidecars. +ZipVoiceConfig load_zipvoice_config( + const std::filesystem::path & model_dir, + const class engine::assets::TensorSource * probe, + const std::filesystem::path * config_path = nullptr); + +ZipVoiceWeights load_zipvoice_weights( + const engine::assets::TensorSource & source, + const std::string & prefix, + const ZipVoiceConfig & config, + ggml_backend_t backend, + core::BackendType backend_type); + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/zipformer.h b/include/engine/community_models/zipvoice/zipformer.h new file mode 100644 index 000000000..239e70111 --- /dev/null +++ b/include/engine/community_models/zipvoice/zipformer.h @@ -0,0 +1,84 @@ +#pragma once + +#include "engine/community_models/zipvoice/weights.h" + +#include +#include +#include + +#include +#include +#include + +namespace engine::models::zipvoice { + +// Raw-ggml graph builders for the two ZipVoice networks. +// +// Physical layouts (ne0 = fastest): +// activations [C, T, B]; attention weights [src, tgt, H, B] +// All softmaxes run over ne0 (= keys). +// +// Memory model: leaves + constants live in a dedicated tensor context with +// their own backend buffer (ggml_backend_alloc_ctx_tensors) so the graph +// arena never aliases them; per-call values are uploaded into that buffer. + +struct ZipVoiceGraphResources { + ggml_context * ctx = nullptr; + ggml_context * tensor_ctx = nullptr; + ggml_backend_buffer_t buffer = nullptr; + ggml_cgraph * graph = nullptr; + ggml_backend_t backend = nullptr; + void * gallocr = nullptr; + ZipVoiceGraphResources() = default; + ~ZipVoiceGraphResources(); + ZipVoiceGraphResources(const ZipVoiceGraphResources &) = delete; + ZipVoiceGraphResources & operator=(const ZipVoiceGraphResources &) = delete; + ZipVoiceGraphResources(ZipVoiceGraphResources && other) noexcept + : ctx(std::exchange(other.ctx, nullptr)), tensor_ctx(std::exchange(other.tensor_ctx, nullptr)), + buffer(std::exchange(other.buffer, nullptr)), graph(std::exchange(other.graph, nullptr)), + backend(std::exchange(other.backend, nullptr)), gallocr(std::exchange(other.gallocr, nullptr)) {} +}; + +struct FmDecoderGraph : ZipVoiceGraphResources { + ggml_tensor * x_cat = nullptr; // leaf [3F, T, B] + ggml_tensor * time_emb = nullptr; // leaf [time_embed_dim] + ggml_tensor * guidance_emb = nullptr; // leaf [time_embed_dim] (distill) + ggml_tensor * pad_bias[8] = {}; // leaves [T_s, 1, 1, 1] (0/-1000) + ggml_tensor * conv_gate[8] = {}; // leaves [1, T_s, 1] (0/1) + ggml_tensor * output = nullptr; // [F, T, B] + int64_t T = 0; + int64_t B = 1; + bool cuda = false; +}; + +struct TextEncoderGraph : ZipVoiceGraphResources { + ggml_tensor * token_ids = nullptr; // leaf [S] i32 + ggml_tensor * pad_bias[1] = {}; // [S, 1, 1, 1] + ggml_tensor * conv_gate[1] = {}; // [1, S, 1] + ggml_tensor * output = nullptr; // [feat_dim, S] + std::vector layer_taps; // per-layer outputs [C, S] + std::vector stage_taps; // first-layer submodule taps + int64_t S = 0; + bool cuda = false; +}; + +// Builds the flow-matching decoder graph. `with_guidance` wires the distill +// guidance-scale embedding input (ignored for the base model). +FmDecoderGraph build_fm_decoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t T, + int64_t B, + bool with_guidance, + bool cuda_backend, + ggml_backend_t backend); + +// Builds the text encoder graph (single stack, no time embedding). +TextEncoderGraph build_text_encoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t S, + bool cuda_backend, + ggml_backend_t backend); + +} // namespace engine::models::zipvoice diff --git a/include/engine/framework/modules/convnext_modules.h b/include/engine/framework/modules/convnext_modules.h new file mode 100644 index 000000000..5244a12a1 --- /dev/null +++ b/include/engine/framework/modules/convnext_modules.h @@ -0,0 +1,18 @@ +#pragma once +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" + +namespace engine::modules { +// Vocos/F5-style non-causal ConvNeXt, logical [batch, frames, channels]. +struct ConvNeXt1dWeights { + DepthwiseConv1dWeights depthwise; + NormWeights norm; + LinearWeights expansion; + LinearWeights projection; + core::TensorValue gamma; +}; +core::TensorValue build_convnext1d( + core::ModuleBuildContext & ctx, const core::TensorValue & input, + const ConvNeXt1dWeights & weights); +} diff --git a/include/engine/framework/modules/vocoders/vocos_vocoder.h b/include/engine/framework/modules/vocoders/vocos_vocoder.h new file mode 100644 index 000000000..c6fa94231 --- /dev/null +++ b/include/engine/framework/modules/vocoders/vocos_vocoder.h @@ -0,0 +1,34 @@ +#pragma once +#include "engine/framework/core/backend.h" +#include "engine/framework/modules/convnext_modules.h" +#include +#include +#include + +namespace engine::modules { +struct VocosBackboneWeights { + Conv1dWeights embed; + NormWeights input_norm, final_norm; + LinearWeights head; + std::vector blocks; +}; +// Shared F5/ZipVoice backbone, logical [batch, frames, mel] -> spectral rows. +core::TensorValue build_vocos_backbone(core::ModuleBuildContext & ctx, + const core::TensorValue & mel, const VocosBackboneWeights & weights); + +// Generalized F5/Vocos mel-24khz graph: Conv1d, ConvNeXt, LN, spectral head. +// Owns backend weights and one runtime graph/ISTFT workspace. The backend is +// borrowed and must outlive this object. Callers serialize decode calls. +class VocosVocoder { +public: + VocosVocoder(const std::string & checkpoint, ggml_backend_t backend, int threads); + ~VocosVocoder(); + VocosVocoder(const VocosVocoder &) = delete; + VocosVocoder & operator=(const VocosVocoder &) = delete; + std::vector decode(const std::vector & mel_rows); + size_t cached_graph_count() const; +private: + class Impl; + std::unique_ptr impl_; +}; +} diff --git a/model_specs/zipvoice.json b/model_specs/zipvoice.json new file mode 100644 index 000000000..77e2fec5e --- /dev/null +++ b/model_specs/zipvoice.json @@ -0,0 +1,232 @@ +{ + "schema_version": 1, + "family": "zipvoice", + "display_name": "ZipVoice", + "description": "Community ZipVoice / ZipVoice-Distill flow-matching TTS with a TTSZipformer backbone (U-Net downsampling stacks, compact relative position attention), duration prediction from prompt ratio, Euler solver with t-shift and classifier-free guidance, and Vocos mel-24kHz vocoder (k2-fsa/ZipVoice).", + "category": "tts", + "status": "community", + "tasks": [ + "tts", + "clone" + ], + "modes": [ + "offline" + ], + "languages": [ + "en", + "zh" + ], + "runtime": { + "tags": [ + "server" + ] + }, + "capabilities": { + "clone": [ + "speaker_reference" + ] + }, + "options": { + "request": [ + { + "name": "reference_text", + "type": "string", + "description": "Transcript matching the reference voice audio; required for zero-shot cloning.", + "required": true + }, + { + "name": "guidance_scale", + "type": "float", + "description": "Classifier-free guidance scale; default 3.0 for ZipVoice-Distill (guidance-scale embedding), 1.0 for ZipVoice (batched CFG). 0 disables guidance.", + "required": false, + "min": 0.0, + "max": 10.0, + "default": 3.0 + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Euler ODE steps; default 8 for ZipVoice-Distill, 16 for ZipVoice.", + "required": false, + "min": 1, + "max": 64, + "default": 8 + }, + { + "name": "t_shift", + "type": "float", + "description": "Shift timesteps toward low SNR (smaller = stronger shift); default 0.5.", + "required": false, + "min": 0.05, + "max": 1.0, + "default": 0.5 + }, + { + "name": "speed", + "type": "float", + "description": "Speech speed multiplier applied through the prompt-duration ratio; default 1.0.", + "required": false, + "min": 0.5, + "max": 2.0, + "default": 1.0 + }, + { + "name": "feat_scale", + "type": "float", + "description": "Feature scale applied to log-mel features; default 0.1 (reference default).", + "required": false, + "min": 0.01, + "max": 1.0, + "default": 0.1 + }, + { + "name": "target_rms", + "type": "float", + "description": "Target RMS for prompt loudness normalization; 0 disables. Default 0.1.", + "required": false, + "min": 0.0, + "max": 1.0, + "default": 0.1 + }, + { + "name": "seed", + "type": "int", + "description": "Noise seed; default 666 (reference default).", + "required": false, + "min": 0, + "default": 666 + }, + { + "name": "lang", + "type": "string", + "description": "espeak language for phonemization; default en-us.", + "required": false, + "default": "en-us" + } + ], + "session": [ + { + "name": "vocos_path", + "type": "string", + "description": "Path to the Vocos vocoder checkpoint (vocos.safetensors or GGUF); required unless bundled in the model GGUF or placed next to the checkpoint.", + "required": false + }, + { + "name": "guidance_scale", + "type": "float", + "description": "Default guidance scale for requests that do not set it.", + "required": false, + "min": 0.0, + "max": 10.0, + "default": 3.0 + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Default Euler steps for requests that do not set it.", + "required": false, + "min": 1, + "max": 64, + "default": 8 + }, + { + "name": "t_shift", + "type": "float", + "description": "Default timestep shift for requests that do not set it.", + "required": false, + "min": 0.05, + "max": 1.0, + "default": 0.5 + }, + { + "name": "espeak_library_path", + "type": "string", + "description": "Path to the espeak-ng shared library (e.g. /opt/homebrew/lib/libespeak-ng.dylib) for tokenizer=espeak.", + "required": false + }, + { + "name": "espeak_data_path", + "type": "string", + "description": "Path to the espeak-ng data directory or package for tokenizer=espeak.", + "required": false + } + ], + "load": [] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "davidxifeng/zipvoice-gguf", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "zipvoice_distill_gguf", + "display_name": "ZipVoice-Distill GGUF (local conversion)", + "description": "Self-contained GGUF: flow-matching model, bundled Vocos vocoder and the embedded text-frontend sidecars (tokens, config, zh tables) in one file. Converted from k2-fsa/ZipVoice with export_zipvoice_zh_dict.py + convert_zipvoice.py.", + "default": true, + "format": "gguf", + "precision": "orig", + "target_directory": "ZipVoice-Distill-GGUF", + "files": [ + "zipvoice-distill-orig.gguf" + ] + }, + { + "id": "zipvoice_distill_q8_0", + "display_name": "ZipVoice-Distill GGUF (Q8_0)", + "description": "Self-contained Q8_0 GGUF with bundled Vocos and embedded text-frontend sidecars.", + "format": "gguf", + "precision": "q8_0", + "target_directory": "ZipVoice-Distill-GGUF", + "files": [ + "zipvoice-distill-q8_0.gguf" + ] + } + ], + "dependencies": [], + "ui": { + "recommended_package": "zipvoice_distill_gguf", + "tags": [ + "TTS", + "Clone" + ], + "docs": [ + "docs/community_models/zipvoice.md" + ] + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "tokens": "model:tokens.txt", + "model_config": "model:model.json" + }, + "optional_files": { + "zh_chars": "model:zh_chars.tsv", + "zh_phrases": "model:zh_phrases.tsv", + "zh_syllables": "model:zh_syllables.tsv", + "zh_jieba_dict": "model:zh_jieba_dict.txt", + "zh_hmm_model": "model:zh_hmm_model.txt" + }, + "tensors": { + "model": { + "source": "weights:", + "prefix": "model" + } + }, + "optional_tensors": { + "vocos_vocoder": { + "source": "weights:", + "prefix": "vocos" + } + } + } + ] +} diff --git a/src/community_models/zipvoice/emilia_tokenizer.cpp b/src/community_models/zipvoice/emilia_tokenizer.cpp new file mode 100644 index 000000000..b9bcc27c5 --- /dev/null +++ b/src/community_models/zipvoice/emilia_tokenizer.cpp @@ -0,0 +1,342 @@ +#include "engine/community_models/zipvoice/emilia_tokenizer.h" + +#include "jieba_segmenter.h" + +#include "engine/framework/audio/espeak_phonemizer.h" +#include "engine/framework/text/chinese_normalization.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +// --------------------------------------------------------------------------- +// UTF-8 helpers + +std::vector split_utf8(const std::string & text) { + std::vector out; + for (size_t i = 0; i < text.size();) { + const auto lead = static_cast(text[i]); + size_t width = 1; + if (lead >= 0xF0) width = 4; + else if (lead >= 0xE0) width = 3; + else if (lead >= 0xC0) width = 2; + out.emplace_back(text, i, std::min(width, text.size() - i)); + i += out.back().size(); + } + return out; +} + +uint32_t utf8_to_codepoint(const std::string & ch) { + const auto lead = static_cast(ch[0]); + if (lead < 0x80) return lead; + if ((lead & 0xE0) == 0xC0) return (uint32_t(lead & 0x1F) << 6) | + (uint32_t(static_cast(ch[1])) & 0x3F); + if ((lead & 0xF0) == 0xE0) return (uint32_t(lead & 0x0F) << 12) | + (uint32_t(static_cast(ch[1])) & 0x3F) << 6 | + (uint32_t(static_cast(ch[2])) & 0x3F); + return (uint32_t(lead & 0x07) << 18) | + (uint32_t(static_cast(ch[1])) & 0x3F) << 12 | + (uint32_t(static_cast(ch[2])) & 0x3F) << 6 | + (uint32_t(static_cast(ch[3])) & 0x3F); +} + +bool is_han_char(uint32_t cp) { + // Reference EmiliaTokenizer::is_chinese checks U+4E00..U+9FA5 only. + return cp >= 0x4E00 && cp <= 0x9FA5; +} + +bool is_alpha_char(uint32_t cp) { + return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z'); +} + +// --------------------------------------------------------------------------- +// Table loading + +std::unordered_map> load_token_table( + const std::filesystem::path & path) { + std::unordered_map> out; + std::ifstream in(path); + if (!in) throw std::runtime_error("zipvoice: missing Chinese table " + path.string()); + std::string line; + while (std::getline(in, line)) { + if (line.empty()) continue; + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + std::vector tokens; + std::istringstream rest(line.substr(tab + 1)); + std::string token; + while (rest >> token) tokens.push_back(token); + out.emplace(line.substr(0, tab), std::move(tokens)); + } + return out; +} + +// --------------------------------------------------------------------------- +// Text shaping (mirrors EmiliaTokenizer::map_punctuations) + +std::string map_punctuations(std::string text) { + static const std::pair kMap[] = { + {"，", ","}, {"。", "."}, {"！", "!"}, {"？", "?"}, {"；", ";"}, + {"：", ":"}, {"、", ","}, {"‘", "'"}, {"“", "\""}, {"”", "\""}, + {"’", "'"}, {"⋯", "…"}, {"···", "…"}, {"・・・", "…"}, {"...", "…"}, + }; + for (const auto & [from, to] : kMap) { + for (size_t pos = 0; (pos = text.find(from, pos)) != std::string::npos;) { + text.replace(pos, from.size(), to); + pos += to.size(); + } + } + return text; +} + +// --------------------------------------------------------------------------- +// Tone sandhi (pypinyin contrib/tone_sandhi.py semantics) +// +// Scope note: the reference only reaches this with a word the pypinyin +// phrases_dict matched (whole-word scope) or a single unmatched character; +// the caller mirrors that by applying it per phrase-match / per character. + +void apply_tone_sandhi(std::vector & tokens, const std::string & han) { + // Syllables are the tokens whose last char is a tone digit. + std::vector finals; + for (size_t i = 0; i < tokens.size(); ++i) { + const auto & t = tokens[i]; + if (!t.empty() && t.back() >= '1' && t.back() <= '5') finals.push_back(i); + } + const size_t n = finals.size(); + if (n == 0) return; + auto tone = [&](size_t s) { return tokens[finals[s]].back(); }; + auto set_tone = [&](size_t s, char d) { tokens[finals[s]].back() = d; }; + + // Third tone: pypinyin counts the TRAILING consecutive run of '3' + // syllables. A run of exactly 2 turns the first '3' in the word into + // '2'; a longer run turns every '3' but the last into '2'. + size_t run = 0; + for (size_t s = n; s > 0; --s) { + if (tone(s - 1) != '3') break; + ++run; + } + if (run == 2) { + for (size_t s = 0; s < n; ++s) { + if (tone(s) == '3') { set_tone(s, '2'); break; } + } + } else if (run > 2) { + size_t seen = 0; + for (size_t s = 0; s < n; ++s) { + if (tone(s) != '3') continue; + if (++seen == run) break; + set_tone(s, '2'); + } + } + + // 不 / 一 (pypinyin's replace('4','2') only fires when the current tone + // is '4'; the non-4th branch FORCES '4'; a final 一 forces '1'). + const auto han_chars = split_utf8(han); + for (size_t s = 0; s < n && s < han_chars.size(); ++s) { + const bool is_bu = han_chars[s] == "不"; + const bool is_yi = han_chars[s] == "一"; + if (!is_bu && !is_yi) continue; + if (s + 1 < n) { + if (tone(s + 1) == '4') { + if (tone(s) == '4') set_tone(s, '2'); + } else { + set_tone(s, '4'); + } + } else { + set_tone(s, is_yi ? '1' : '4'); + } + } +} + +} // namespace + +// --------------------------------------------------------------------------- +// EmiliaTokenizer + +EmiliaTokenizer::EmiliaTokenizer( + const TablePaths & tables, + const std::unordered_map & vocab, + const EspeakConfig & espeak) + : vocab_(vocab), tables_(tables), espeak_(espeak) { + chars_ = load_token_table(tables_.chars); + phrases_ = load_token_table(tables_.phrases); + syllables_ = load_token_table(tables_.syllables); +} + +EmiliaTokenizer::~EmiliaTokenizer() = default; + +std::vector EmiliaTokenizer::encode(const std::string & raw_text) const { + // phone string -> id + auto emit = [&](const std::vector & phones, std::vector & ids) { + for (const auto & phone : phones) { + if (const auto it = vocab_.find(phone); it != vocab_.end()) { + ids.push_back(it->second); + } + } + }; + + // Segment into (text, lang) runs: zh (Han), en (ASCII letters), other. + // "other" attaches to the current language; a leading "other" run takes + // the language of the following text. + struct Segment { std::string text; char lang; }; + std::vector segments; + { + const auto chars = split_utf8(raw_text); + char lang = 'o'; + std::string current; + for (const auto & ch : chars) { + const auto cp = utf8_to_codepoint(ch); + const char type = is_han_char(cp) ? 'z' : (is_alpha_char(cp) ? 'e' : 'o'); + if (current.empty()) { + current = ch; + lang = type; + } else if (lang == 'o') { + current += ch; + lang = type; + } else if (type == lang || type == 'o') { + current += ch; + } else { + segments.push_back({current, lang}); + current = ch; + lang = type; + } + } + if (!current.empty()) segments.push_back({current, lang}); + } + + // Split / [tag] parts out of every segment. + struct Part { std::string text; char lang; bool bracket; }; + std::vector parts; + for (auto & seg : segments) { + std::string pending; + for (size_t i = 0; i < seg.text.size();) { + const char c = seg.text[i]; + if (c == '<' || c == '[') { + const char close = c == '<' ? '>' : ']'; + const auto end = seg.text.find(close, i + 1); + if (end != std::string::npos) { + if (!pending.empty()) { + parts.push_back({pending, seg.lang, false}); + pending.clear(); + } + parts.push_back({seg.text.substr(i, end - i + 1), seg.lang, true}); + i = end + 1; + continue; + } + } + pending += c; + ++i; + } + if (!pending.empty()) parts.push_back({pending, seg.lang, false}); + } + + std::vector ids; + for (const auto & part : parts) { + if (part.bracket) { + if (part.text.front() == '<' && part.text.back() == '>') { + // Inline pinyin override, e.g. -> "w0 o3". + const std::string syllable = part.text.substr(1, part.text.size() - 2); + if (!syllable.empty() && std::isalpha(static_cast(syllable.front())) && + syllable.back() >= '1' && syllable.back() <= '5') { + if (const auto it = syllables_.find(syllable); it != syllables_.end()) { + emit(it->second, ids); + } + } + } else { + // [tag]: the whole bracketed string is one phone (usually OOV). + emit({part.text}, ids); + } + continue; + } + if (part.lang == 'e') { + // English segment: eSpeak-ng phonemization, one phone per + // codepoint (matches the audio.cpp espeak frontend convention). + // eSpeak treats sentence punctuation as clause markers and drops + // it from the phoneme stream, but the reference keeps it as + // tokens, so re-append trailing punctuation here. + static const std::string kTrailingPunct = ".,!?;:"; + std::string text = part.text; + std::string trailing; + while (!text.empty() && kTrailingPunct.find(text.back()) != std::string::npos) { + trailing.insert(trailing.begin(), text.back()); + text.pop_back(); + } + if (!phonemizer_) { + phonemizer_ = std::make_unique( + std::filesystem::path{espeak_.library_path}, + std::filesystem::path{espeak_.data_path}, + std::vector{espeak_.lang.empty() ? "en-us" : espeak_.lang}); + } + if (!text.empty()) { + bool all_space = true; + for (const char c : text) { + if (c != ' ') { all_space = false; break; } + } + if (all_space) { + for (const auto & ch : split_utf8(text)) emit({ch}, ids); + } else { + emit(split_utf8(phonemizer_->phonemize(text, 2)), ids); + } + } + for (const auto & ch : split_utf8(trailing)) emit({ch}, ids); + continue; + } + if (part.lang != 'z') continue; // "other"-only segments are dropped + + // Chinese segment: digit normalization, then jieba word segmentation + // (the model-local MixSegment port, identical to python jieba.cut). + // Each word is one reading + sandhi unit, exactly like the reference + // pipeline lazy_pinyin(jieba.cut(text), tone_sandhi=True). + const std::string normalized = text::normalize_chinese_text(part.text); + std::vector words; + { + std::lock_guard lock(segmenter_mutex_); + if (!segmenter_) { + if (!std::filesystem::is_regular_file(tables_.jieba_dict) || + !std::filesystem::is_regular_file(tables_.hmm_model)) { + throw std::runtime_error( + "zipvoice: jieba dictionaries missing (expected zh_jieba_dict.txt " + "and zh_hmm_model.txt in the package or next to it); run " + "tools/community_models/export_zipvoice_zh_dict.py"); + } + segmenter_ = std::make_unique( + tables_.jieba_dict, tables_.hmm_model); + } + words = segmenter_->cut(normalized); + } + for (const auto & word : words) { + const auto word_chars = split_utf8(word); + bool has_han = false; + for (const auto & ch : word_chars) { + if (is_han_char(utf8_to_codepoint(ch))) has_han = true; + } + if (!has_han) { + for (const auto & ch : word_chars) emit({ch}, ids); + continue; + } + std::vector tokens; + if (const auto it = phrases_.find(word); it != phrases_.end()) { + tokens = it->second; + } else { + // Unknown word: per-character default readings; characters + // without a pinyin entry are skipped, like the reference. + for (const auto & ch : word_chars) { + if (const auto cit = chars_.find(ch); cit != chars_.end()) { + tokens.insert(tokens.end(), cit->second.begin(), cit->second.end()); + } + } + } + apply_tone_sandhi(tokens, word); + emit(tokens, ids); + } + } + return ids; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/jieba_segmenter.cpp b/src/community_models/zipvoice/jieba_segmenter.cpp new file mode 100644 index 000000000..860ce3f35 --- /dev/null +++ b/src/community_models/zipvoice/jieba_segmenter.cpp @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: MIT +// +// Focused port of cppjieba's MixSegment for ZipVoice. Derived from: +// cppjieba, Copyright (c) 2013 Yanyi Wu (MIT) +// jieba, Copyright (c) 2012 Sun Junyi (MIT) +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Only the pieces the ZipVoice Chinese frontend needs are kept: the PreFilter +// separator pass, the maximum-probability DAG segmentation over the jieba +// dictionary, and the four-state BMES HMM used for runs of dictionary-missed +// single characters. The dictionary files are model resources embedded in the +// GGUF, not a vendored library. + +#include "jieba_segmenter.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +// cppjieba Utils.hpp MIN_DOUBLE, used as the "impossible" log-probability. +constexpr double kMinDouble = -3.14e+100; + +// cppjieba HMMModel::STATUS_SUM order: 0=B, 1=E, 2=M, 3=S. +constexpr size_t kStateE = 1; +constexpr size_t kStateS = 3; + +// cppjieba Utils.hpp Trim. +void trim(std::string & s) { + auto not_space = [](unsigned char c) { return !std::isspace(c); }; + s.erase(s.begin(), std::find_if(s.begin(), s.end(), not_space)); + s.erase(std::find_if(s.rbegin(), s.rend(), not_space).base(), s.end()); +} + +size_t count_codepoints(const std::string & s) { + size_t n = 0; + for (unsigned char c : s) { + if ((c & 0xC0) != 0x80) ++n; + } + return n; +} + +} // namespace + +JiebaSegmenter::JiebaSegmenter(const std::filesystem::path & dict_path, + const std::filesystem::path & hmm_path) { + // --- dictionary: , weight = log(freq / freq_sum) --- + std::ifstream dict(dict_path); + if (!dict) { + throw std::runtime_error("zipvoice: cannot open jieba dictionary " + dict_path.string()); + } + std::vector> entries; + entries.reserve(400000); + double freq_sum = 0.0; + size_t max_len = 1; + std::string line; + while (std::getline(dict, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + const size_t first = line.find(' '); + if (first == std::string::npos) continue; + const size_t second = line.find(' ', first + 1); + if (second == std::string::npos) continue; + std::string word = line.substr(0, first); + const double freq = std::strtod(line.c_str() + first + 1, nullptr); + freq_sum += freq; + max_len = std::max(max_len, count_codepoints(word)); + entries.emplace_back(std::move(word), freq); + } + if (entries.empty() || !(freq_sum > 0.0)) { + throw std::runtime_error("zipvoice: empty jieba dictionary " + dict_path.string()); + } + weights_.reserve(entries.size() * 2); + double min_weight = std::numeric_limits::infinity(); + for (const auto & [word, freq] : entries) { + const double weight = std::log(freq / freq_sum); + weights_[word] = weight; + min_weight = std::min(min_weight, weight); + } + min_weight_ = min_weight; + max_word_len_ = max_len; + + // --- HMM: start/trans/emit tables (cppjieba hmm_model.utf8) --- + std::ifstream hmm(hmm_path); + if (!hmm) { + throw std::runtime_error("zipvoice: cannot open jieba HMM model " + hmm_path.string()); + } + const auto next_content_line = [&](std::string & out) { + while (std::getline(hmm, out)) { + trim(out); + if (!out.empty() && out[0] != '#') return true; + } + return false; + }; + const auto parse_doubles = [](const std::string & text, double * out, size_t count) { + std::istringstream stream(text); + for (size_t i = 0; i < count; ++i) { + if (!(stream >> out[i])) { + throw std::runtime_error("zipvoice: malformed jieba HMM model"); + } + } + }; + if (!next_content_line(line)) throw std::runtime_error("zipvoice: truncated jieba HMM model"); + parse_doubles(line, start_prob_, 4); + for (size_t i = 0; i < 4; ++i) { + if (!next_content_line(line)) throw std::runtime_error("zipvoice: truncated jieba HMM model"); + parse_doubles(line, trans_prob_[i], 4); + } + for (size_t state = 0; state < 4; ++state) { + if (!next_content_line(line)) throw std::runtime_error("zipvoice: truncated jieba HMM model"); + size_t pos = 0; + while (pos < line.size()) { + const size_t comma = line.find(',', pos); + const std::string entry = line.substr( + pos, comma == std::string::npos ? std::string::npos : comma - pos); + const size_t colon = entry.find(':'); + if (colon != std::string::npos && colon > 0) { + const auto runes = decode(entry.substr(0, colon)); + if (!runes.empty()) { + emit_prob_[state][runes.front().cp] = + std::strtod(entry.c_str() + colon + 1, nullptr); + } + } + if (comma == std::string::npos) break; + pos = comma + 1; + } + } +} + +std::vector JiebaSegmenter::decode(const std::string & text) { + std::vector runes; + runes.reserve(text.size() / 2 + 1); + for (size_t i = 0; i < text.size();) { + const auto lead = static_cast(text[i]); + uint32_t cp = lead; + uint32_t len = 1; + if (lead < 0x80) { + cp = lead; + } else if ((lead & 0xE0) == 0xC0 && i + 1 < text.size()) { + cp = (uint32_t(lead & 0x1F) << 6) | + (uint32_t(static_cast(text[i + 1])) & 0x3F); + len = 2; + } else if ((lead & 0xF0) == 0xE0 && i + 2 < text.size()) { + cp = (uint32_t(lead & 0x0F) << 12) | + (uint32_t(static_cast(text[i + 1])) & 0x3F) << 6 | + (uint32_t(static_cast(text[i + 2])) & 0x3F); + len = 3; + } else if ((lead & 0xF8) == 0xF0 && i + 3 < text.size()) { + cp = (uint32_t(lead & 0x07) << 18) | + (uint32_t(static_cast(text[i + 1])) & 0x3F) << 12 | + (uint32_t(static_cast(text[i + 2])) & 0x3F) << 6 | + (uint32_t(static_cast(text[i + 3])) & 0x3F); + len = 4; + } + runes.push_back({cp, static_cast(i), len}); + i += len; + } + return runes; +} + +std::string JiebaSegmenter::slice(const std::string & text, const std::vector & runes, + size_t begin, size_t end_inclusive) { + const size_t start = runes[begin].offset; + const size_t end = end_inclusive + 1 < runes.size() + ? runes[end_inclusive + 1].offset + : text.size(); + return text.substr(start, end - start); +} + +bool JiebaSegmenter::is_separator(uint32_t cp) { + // cppjieba SegmentBase SPECIAL_SEPARATORS: " \t\n，。". + return cp == 0x20 || cp == 0x09 || cp == 0x0A || cp == 0xFF0C || cp == 0x3002; +} + +size_t JiebaSegmenter::sequential_letter_rule(const std::vector & runes, + size_t begin, size_t end) { + uint32_t x = runes[begin].cp; + if (!(('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z'))) return begin; + ++begin; + while (begin != end) { + x = runes[begin].cp; + if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) ++begin; + else break; + } + if (begin != end && runes[begin].cp == '.') { + if (begin + 1 != end && runes[begin + 1].cp >= '0' && runes[begin + 1].cp <= '9') { + ++begin; + while (begin != end && runes[begin].cp >= '0' && runes[begin].cp <= '9') ++begin; + } + } + return begin; +} + +size_t JiebaSegmenter::numbers_rule(const std::vector & runes, size_t begin, size_t end) { + uint32_t x = runes[begin].cp; + if (!('0' <= x && x <= '9')) return begin; + ++begin; + while (begin != end) { + x = runes[begin].cp; + if (('0' <= x && x <= '9') || ('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) ++begin; + else break; + } + if (begin != end && runes[begin].cp == '.') { + if (begin + 1 != end && runes[begin + 1].cp >= '0' && runes[begin + 1].cp <= '9') { + ++begin; + while (begin != end && runes[begin].cp >= '0' && runes[begin].cp <= '9') ++begin; + } + } + return begin; +} + +double JiebaSegmenter::emit_probability(size_t state, uint32_t cp) const { + const auto & table = emit_prob_[state]; + const auto it = table.find(cp); + return it == table.end() ? kMinDouble : it->second; +} + +void JiebaSegmenter::viterbi_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const { + const size_t X = end - begin; + if (X == 0) return; + std::vector weight(4 * X); + std::vector path(4 * X); + for (size_t y = 0; y < 4; ++y) { + weight[y * X] = start_prob_[y] + emit_probability(y, runes[begin].cp); + path[y * X] = -1; + } + for (size_t x = 1; x < X; ++x) { + for (size_t y = 0; y < 4; ++y) { + const size_t now = x + y * X; + weight[now] = kMinDouble; + path[now] = static_cast(kStateE); + const double emit = emit_probability(y, runes[begin + x].cp); + for (size_t prev = 0; prev < 4; ++prev) { + const double candidate = weight[x - 1 + prev * X] + trans_prob_[prev][y] + emit; + if (candidate > weight[now]) { + weight[now] = candidate; + path[now] = static_cast(prev); + } + } + } + } + const double end_e = weight[(X - 1) + kStateE * X]; + const double end_s = weight[(X - 1) + kStateS * X]; + int state = end_e >= end_s ? static_cast(kStateE) : static_cast(kStateS); + std::vector status(X); + for (size_t x = X; x-- > 0;) { + status[x] = state; + state = path[x + static_cast(state) * X]; + } + size_t left = begin; + for (size_t i = 0; i < X; ++i) { + if (status[i] % 2 != 0) { // E or S ends a word + words.push_back(slice(text, runes, left, begin + i)); + left = begin + i + 1; + } + } +} + +void JiebaSegmenter::hmm_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const { + size_t left = begin; + size_t right = begin; + while (right < end) { + if (runes[right].cp < 0x80) { + if (left != right) viterbi_cut(text, runes, left, right, words); + left = right; + size_t next = sequential_letter_rule(runes, left, end); + if (next == left) next = numbers_rule(runes, left, end); + if (next == left) next = left + 1; + right = next; + words.push_back(slice(text, runes, left, right - 1)); + left = right; + } else { + ++right; + } + } + if (left != right) viterbi_cut(text, runes, left, right, words); +} + +void JiebaSegmenter::segment_range(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const { + const size_t n = end - begin; + if (n == 0) return; + struct Candidate { + size_t end_index; + double weight; + }; + // MP DAG candidates: the single character is always a candidate (unknown + // characters fall back to the dictionary's minimum weight, like cppjieba); + // longer candidates exist only when the whole span is a dictionary word. + std::vector> nexts(n); + for (size_t k = 0; k < n; ++k) { + const size_t i = begin + k; + const std::string single = slice(text, runes, i, i); + const auto single_it = weights_.find(single); + nexts[k].push_back({k, single_it != weights_.end() ? single_it->second : min_weight_}); + for (size_t len = 2; len <= max_word_len_ && k + len <= n; ++len) { + const auto it = weights_.find(slice(text, runes, i, i + len - 1)); + if (it != weights_.end()) nexts[k].push_back({k + len - 1, it->second}); + } + } + // Maximum-probability route, right to left (cppjieba CalcDP). + std::vector best_weight(n + 1, 0.0); + std::vector best_len(n, 1); + for (size_t k = n; k-- > 0;) { + double best = kMinDouble; + size_t length = 1; + for (const auto & candidate : nexts[k]) { + const double value = candidate.weight + + (candidate.end_index + 1 < n ? best_weight[candidate.end_index + 1] : 0.0); + if (value > best) { + best = value; + length = candidate.end_index - k + 1; + } + } + best_weight[k] = best; + best_len[k] = length; + } + // MP words as rune ranges. + std::vector> mp_words; + for (size_t k = 0; k < n;) { + const size_t length = best_len[k]; + mp_words.push_back({begin + k, begin + k + length - 1}); + k += length; + } + // MixSegment: keep multi-character words; re-cut each run of consecutive + // single-character words with the HMM. + for (size_t r = 0; r < mp_words.size();) { + if (mp_words[r].first != mp_words[r].second) { + words.push_back(slice(text, runes, mp_words[r].first, mp_words[r].second)); + ++r; + continue; + } + size_t run_end = r; + while (run_end < mp_words.size() && mp_words[run_end].first == mp_words[run_end].second) { + ++run_end; + } + hmm_cut(text, runes, mp_words[r].first, mp_words[run_end - 1].first + 1, words); + r = run_end; + } +} + +std::vector JiebaSegmenter::cut(const std::string & text) const { + std::vector words; + const auto runes = decode(text); + if (runes.empty()) return words; + // PreFilter: separators are single-character words; the runs between them + // are segmented independently. + size_t i = 0; + while (i < runes.size()) { + if (is_separator(runes[i].cp)) { + words.push_back(slice(text, runes, i, i)); + ++i; + continue; + } + size_t j = i; + while (j < runes.size() && !is_separator(runes[j].cp)) ++j; + segment_range(text, runes, i, j, words); + i = j; + } + return words; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/jieba_segmenter.h b/src/community_models/zipvoice/jieba_segmenter.h new file mode 100644 index 000000000..941c3541e --- /dev/null +++ b/src/community_models/zipvoice/jieba_segmenter.h @@ -0,0 +1,67 @@ +#pragma once + +// SPDX-License-Identifier: MIT +// +// Minimal Jieba word segmentation for the ZipVoice Chinese frontend. +// +// This is a focused port of the parts of cppjieba (Copyright (c) 2013 Yanyi +// Wu, MIT) and jieba (Copyright (c) 2012 Sun Junyi, MIT) that the frontend +// needs: the PreFilter separator pass, the maximum-probability DAG +// segmentation over jieba.dict.utf8, and the four-state BMES HMM over +// hmm_model.utf8. It reads the same dictionary files that are embedded in the +// model GGUF, so no language-specific library is vendored. +// +// cppjieba: https://github.com/yanyiwu/cppjieba (MIT) +// jieba: https://github.com/fxsjy/jieba (MIT) + +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { + +// Word segmentation equivalent to cppjieba::MixSegment (HMM enabled), which in +// turn matches python jieba.cut for the same dictionaries. +class JiebaSegmenter { +public: + JiebaSegmenter(const std::filesystem::path & dict_path, + const std::filesystem::path & hmm_path); + + // Word boundaries for `text` (UTF-8). Separators become single-character + // words; runs of dictionary-missed single characters are re-segmented with + // the BMES HMM, exactly like MixSegment. + std::vector cut(const std::string & text) const; + +private: + struct Rune { + uint32_t cp = 0; + uint32_t offset = 0; // byte offset in the source string + uint32_t len = 0; // byte length + }; + + static std::vector decode(const std::string & text); + static std::string slice(const std::string & text, const std::vector & runes, + size_t begin, size_t end_inclusive); + static bool is_separator(uint32_t cp); + static size_t sequential_letter_rule(const std::vector & runes, size_t begin, size_t end); + static size_t numbers_rule(const std::vector & runes, size_t begin, size_t end); + // MP DAG + HMM over the rune range [begin, end). + void segment_range(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const; + void hmm_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const; + void viterbi_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const; + double emit_probability(size_t state, uint32_t cp) const; + + std::unordered_map weights_; // word -> log(freq/freq_sum) + double min_weight_ = 0.0; + size_t max_word_len_ = 1; + double start_prob_[4] = {}; + double trans_prob_[4][4] = {}; + std::unordered_map emit_prob_[4]; +}; + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/session.cpp b/src/community_models/zipvoice/session.cpp new file mode 100644 index 000000000..87d52b4b8 --- /dev/null +++ b/src/community_models/zipvoice/session.cpp @@ -0,0 +1,278 @@ +#include "engine/community_models/zipvoice/session.h" + +#include "engine/community_models/zipvoice/synthesize.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/model_spec/package.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/text/chunking.h" +#include "engine/framework/runtime/spec_backed_model.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +constexpr const char * kFamily = "zipvoice"; + +struct ZipVoiceAssets { + assets::ResourceBundle resources; + std::filesystem::path checkpoint; // .gguf or safetensors dev dir + std::filesystem::path model_dir; + // session defaults (mutable only during session construction) + int num_steps = 8; + float guidance_scale = 3.0F; + float t_shift = 0.5F; +}; + +const runtime::AudioBuffer * reference_audio(const runtime::TaskRequest & request) { + if (request.voice.has_value() && + request.voice->speaker.has_value() && + request.voice->speaker->audio.has_value()) { + return &*request.voice->speaker->audio; + } + return request.audio_input.has_value() ? &*request.audio_input : nullptr; +} + +std::filesystem::path find_checkpoint(const std::filesystem::path & model_path) { + namespace fs = std::filesystem; + if (fs::is_regular_file(model_path)) { + return model_path; + } + std::vector ggufs, safetensors; + for (const auto & entry : fs::directory_iterator(model_path)) { + const auto ext = entry.path().extension(); + // skip the vocoder package when it is staged in the same directory + if (ext == ".gguf") ggufs.push_back(entry.path()); + else if (ext == ".safetensors") safetensors.push_back(entry.path()); + } + std::sort(ggufs.begin(), ggufs.end()); + std::sort(safetensors.begin(), safetensors.end()); + if (!ggufs.empty()) return ggufs.back(); + if (!safetensors.empty()) return safetensors.back(); + throw std::runtime_error( + "zipvoice: no .gguf/.safetensors checkpoint found in " + model_path.string()); +} + +bool tensor_file_has_namespace(const std::filesystem::path & path, const char * namespace_name) { + try { + const auto source = assets::open_tensor_source(path); + const std::string prefix = std::string(namespace_name) + "/backbone.embed.weight"; + const std::string prefix_dots = std::string(namespace_name) + ".backbone.embed.weight"; + return source->has_tensor(prefix) || source->has_tensor(prefix_dots); + } catch (...) { + return false; + } +} + +std::optional find_tensor_file(const std::filesystem::path & dir) { + namespace fs = std::filesystem; + if (!fs::is_directory(dir)) return std::nullopt; + for (const char * ext : {".safetensors", ".gguf"}) { + for (const auto & entry : fs::directory_iterator(dir)) { + if (entry.path().extension() == ext) return entry.path(); + } + } + return std::nullopt; +} + +std::shared_ptr load_assets(const std::filesystem::path & model_path) { + auto holder = std::make_shared(); + // Spec-resolved bundle: registered sidecars (tokens, model_config and the + // optional zh_* frontend tables) resolve to materialized GGUF-embedded + // copies or development-directory files; audio.cpp community models + // (audio8_asr, vibeasr, glm_tts, ...) follow the same pattern. + holder->resources = engine::model_spec::load_resource_bundle_for_family(model_path, kFamily); + holder->checkpoint = find_checkpoint(model_path); + holder->model_dir = + std::filesystem::is_regular_file(model_path) ? model_path.parent_path() : model_path; + return holder; +} + +class ZipVoiceSession final : public runtime::IOfflineVoiceTaskSession { +public: + ZipVoiceSession( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) + : task_kind_(task.task), + run_mode_(task.mode), + assets_(std::move(assets)), + contract_(std::move(contract)) { + if (assets_ == nullptr) { + throw std::runtime_error("zipvoice session requires assets"); + } + if (contract_ == nullptr) { + throw std::runtime_error("zipvoice session requires a model contract"); + } + runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "ZipVoice"); + // Vocos resolution order (f5_tts pattern): + // 1. zipvoice.vocos_path session option + // 2. bundled "vocos" namespace in the GGUF checkpoint + // 3. vocos.safetensors next to the checkpoint + // 4. vocos-mel-24khz package next to the model directory + namespace fs = std::filesystem; + if (const auto v = runtime::find_option(options.options, {"zipvoice.vocos_path", "vocos_path"})) { + vocos_path_ = *v; + } else if (assets_->checkpoint.extension() == ".gguf" && + tensor_file_has_namespace(assets_->checkpoint, "vocos")) { + vocos_path_ = assets_->checkpoint.string(); + } else { + const fs::path sibling = assets_->checkpoint.parent_path() / "vocos.safetensors"; + if (fs::exists(sibling)) { + vocos_path_ = sibling.string(); + } else if (const auto pkg = find_tensor_file( + assets_->model_dir.parent_path() / "vocos-mel-24khz")) { + vocos_path_ = pkg->string(); + } + } + if (vocos_path_.empty()) { + throw std::runtime_error( + "zipvoice: no vocos vocoder found; install the vocos-mel-24khz " + "package or set session option zipvoice.vocos_path"); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.num_inference_steps", "num_inference_steps"})) { + num_steps_ = std::stoi(*v); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.guidance_scale", "guidance_scale"})) { + guidance_scale_ = std::stof(*v); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.t_shift", "t_shift"})) { + t_shift_ = std::stof(*v); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.espeak_data_path", "espeak_data_path"})) { + espeak_data_path_ = *v; + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.espeak_library_path", "espeak_library_path"})) { + espeak_library_path_ = *v; + } + device_.backend_type = options.backend.type; + device_.device_index = options.backend.device; + device_.threads = options.backend.threads; + } + + std::string family() const noexcept override { return kFamily; } + runtime::VoiceTaskKind task_kind() const noexcept override { return task_kind_; } + runtime::RunMode run_mode() const noexcept override { return run_mode_; } + + void prepare(const runtime::SessionPreparationRequest & request) override { + (void) request; + } + + runtime::TaskResult run(const runtime::TaskRequest & request) override { + const auto budget = text::parse_text_chunk_size_override(request.options).value_or(128); + if (budget <= 0) throw std::invalid_argument("zipvoice: text_chunk_size must be positive"); + const auto mode = text::parse_text_chunk_mode_override(request.options).value_or(text::TextChunkMode::Default); + const auto chunks = runtime::chunk_text_request(request, budget, mode); + runtime::AudioBuffer merged; + for (const auto & chunk : chunks) { + auto result = run_chunk(chunk); + runtime::append_audio_buffer(merged, *result.audio_output); + } + if (merged.samples.empty()) throw std::invalid_argument("zipvoice: empty text chunks"); + runtime::TaskResult result; + result.audio_output = std::move(merged); + return result; + } + + runtime::TaskResult run_chunk(const runtime::TaskRequest & request) { + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("zipvoice requires input text"); + } + const runtime::AudioBuffer * ref = reference_audio(request); + if (ref == nullptr || ref->samples.empty()) { + throw std::runtime_error( + "zipvoice requires reference voice audio (voice preset or voice_ref)"); + } + const auto ref_text_it = request.options.find("reference_text"); + if (ref_text_it == request.options.end() || ref_text_it->second.empty()) { + throw std::runtime_error( + "zipvoice requires reference_text (transcript of the reference audio)"); + } + + ZipVoiceSynthesisRequest req; + req.text = request.text_input->text; + req.ref_text = ref_text_it->second; + req.ref_audio = ref->samples; + req.ref_sample_rate = ref->sample_rate; + req.ref_channels = ref->channels; + req.tokenizer = "emilia"; // fixed frontend: zh/en/mixed via the Emilia pipeline + req.espeak_library_path = espeak_library_path_; + req.espeak_data_path = espeak_data_path_; + if (const auto v = runtime::find_option(request.options, {"lang"})) req.lang = *v; + req.num_steps = num_steps_; + req.guidance_scale = guidance_scale_; + req.t_shift = t_shift_; + if (const auto v = runtime::find_option(request.options, {"num_inference_steps"})) { + req.num_steps = std::stoi(*v); + } + if (const auto v = runtime::find_option(request.options, {"guidance_scale", "cfg_strength"})) { + req.guidance_scale = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"t_shift"})) { + req.t_shift = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"speed"})) { + req.speed = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"feat_scale"})) { + req.feat_scale = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"target_rms"})) { + req.target_rms = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"seed"})) { + req.seed = static_cast(std::stoul(*v)); + req.fixed_seed = true; + } + + auto out = zipvoice_synthesize( + assets_->checkpoint.string(), vocos_path_, req, device_, &assets_->resources); + + runtime::TaskResult result; + runtime::AudioBuffer audio; + audio.sample_rate = out.sample_rate; + audio.channels = 1; + audio.samples = std::move(out.audio); + result.audio_output = std::move(audio); + return result; + } + +private: + runtime::VoiceTaskKind task_kind_; + runtime::RunMode run_mode_; + std::shared_ptr assets_; + std::shared_ptr contract_; + std::string vocos_path_; + int num_steps_ = 8; + float guidance_scale_ = 3.0F; + float t_shift_ = 0.5F; + std::string espeak_library_path_; + std::string espeak_data_path_; + ZipVoiceComputeDevice device_; +}; + +} // namespace + +std::shared_ptr make_zipvoice_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = std::string(kFamily); + config.load_assets = load_assets; + config.create_session = []( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, options, std::move(assets), std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/synthesize.cpp b/src/community_models/zipvoice/synthesize.cpp new file mode 100644 index 000000000..bb5a33ff5 --- /dev/null +++ b/src/community_models/zipvoice/synthesize.cpp @@ -0,0 +1,1030 @@ +#include "engine/community_models/zipvoice/synthesize.h" +#include "engine/framework/runtime/cache_slots.h" +#include "engine/framework/modules/vocoders/vocos_vocoder.h" + +#include "engine/community_models/zipvoice/emilia_tokenizer.h" +#include "engine/community_models/zipvoice/weights.h" +#include "engine/community_models/zipvoice/zipformer.h" + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/audio/espeak_phonemizer.h" +#include "engine/framework/audio/conversion.h" + +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +// --------------------------------------------------------------------------- +// log-mel (VocosFbank parity: torchaudio MelSpectrogram 24k/1024/256/100, +// power=1, htk scale, f_max=12000, log clamp 1e-7, lhotse frame alignment) + +constexpr float kPi = 3.14159265358979323846F; +constexpr int kSampleRate = 24000; +constexpr int kNfft = 1024; +constexpr int kHop = 256; +constexpr int kNMel = 100; +constexpr float kFMin = 0.0F; +constexpr float kFMax = 12000.0F; + +float hz_to_mel_htk(float hz) { return 2595.0F * std::log10(1.0F + hz / 700.0F); } +float mel_to_hz_htk(float mel) { return 700.0F * (std::pow(10.0F, mel / 2595.0F) - 1.0F); } + +std::vector mel_filterbank() { + std::vector fb; + { + const int n_freqs = kNfft / 2 + 1; + const float m_min = hz_to_mel_htk(kFMin); + const float m_max = hz_to_mel_htk(kFMax); + std::vector mels(kNMel + 2); + for (int i = 0; i < kNMel + 2; ++i) { + mels[i] = mel_to_hz_htk(m_min + (m_max - m_min) * i / (kNMel + 1)); + } + fb.assign(static_cast(n_freqs) * kNMel, 0.0F); + for (int m = 0; m < kNMel; ++m) { + const float lo = mels[m]; + const float mid = mels[m + 1]; + const float hi = mels[m + 2]; + for (int f = 0; f < n_freqs; ++f) { + const float freq = static_cast(f) * kSampleRate / kNfft; + if (freq <= lo || freq >= hi) continue; + const float w = freq <= mid + ? (freq - lo) / (mid - lo) + : (hi - freq) / (hi - mid); + fb[static_cast(f) * kNMel + m] = w; + } + } + } + return fb; +} + +void fft_inplace(std::vector & re, std::vector & im, bool inverse) { + const size_t n = re.size(); + for (size_t i = 1, j = 0; i < n; ++i) { + size_t bit = n >> 1; + for (; j & bit; bit >>= 1) j ^= bit; + j ^= bit; + if (i < j) { + std::swap(re[i], re[j]); + std::swap(im[i], im[j]); + } + } + for (size_t len = 2; len <= n; len <<= 1) { + const float ang = static_cast(2.0 * kPi / static_cast(len)) * (inverse ? 1 : -1); + for (size_t i = 0; i < n; i += len) { + for (size_t k = 0; k < len / 2; ++k) { + const float wr = std::cos(ang * static_cast(k)); + const float wi = std::sin(ang * static_cast(k)); + const size_t a = i + k; + const size_t b = i + k + len / 2; + const float vr = re[b] * wr - im[b] * wi; + const float vi = re[b] * wi + im[b] * wr; + const float ur = re[a]; + const float ui = im[a]; + re[a] = ur + vr; + im[a] = ui + vi; + re[b] = ur - vr; + im[b] = ui - vi; + } + } + } + if (inverse) { + for (size_t i = 0; i < n; ++i) { + re[i] /= static_cast(n); + im[i] /= static_cast(n); + } + } +} + +} // namespace + +std::vector zipvoice_logmel(const std::vector & wav) { + if (wav.size() <= kNfft / 2) { + throw std::invalid_argument("zipvoice: reference audio must contain more than 512 samples at 24 kHz"); + } + const int n_freqs = kNfft / 2 + 1; + // lhotse compute_num_frames: (samples + hop/2) / hop + const int frames = std::max(1, (static_cast(wav.size()) + kHop / 2) / kHop); + std::vector hann(kNfft); + for (int i = 0; i < kNfft; ++i) { + hann[i] = 0.5F * (1.0F - std::cos(2.0F * static_cast(kPi) * i / kNfft)); + } + const auto & fb = mel_filterbank(); + std::vector mel(static_cast(frames) * kNMel); // [t, mel] row-major + std::vector re(kNfft), im(kNfft), mag(n_freqs); + for (int t = 0; t < frames; ++t) { + std::fill(re.begin(), re.end(), 0.0F); + std::fill(im.begin(), im.end(), 0.0F); + const int start = (t - 2) * kHop; // torchaudio center=True alignment + for (int i = 0; i < kNfft; ++i) { + int r = start + i; + if (r < 0) r = -r; + if (r >= static_cast(wav.size())) r = 2 * static_cast(wav.size()) - 2 - r; + r = std::clamp(r, 0, static_cast(wav.size()) - 1); + re[i] = wav[static_cast(r)] * hann[i]; + } + fft_inplace(re, im, false); + for (int f = 0; f < n_freqs; ++f) { + mag[f] = std::sqrt(re[f] * re[f] + im[f] * im[f]); + } + for (int m = 0; m < kNMel; ++m) { + float acc = 0.0F; + for (int f = 0; f < n_freqs; ++f) { + acc += fb[static_cast(f) * kNMel + m] * mag[f]; + } + mel[static_cast(t) * kNMel + m] = std::log(std::max(acc, 1e-7F)); + } + } + return mel; +} + +std::vector zipvoice_vocos_decode(const std::string & path, const std::vector & mel) { + const auto free_backend = [](ggml_backend_t b) { ggml_backend_free(b); }; + std::unique_ptr backend( + core::init_backend({core::BackendType::Cpu, 0, 4}), free_backend); + modules::VocosVocoder vocos(path, backend.get(), 4); + return vocos.decode(mel); +} + +namespace { + +// --------------------------------------------------------------------------- +// Session/device-owned weights and bounded runtime graph slots + +std::unordered_map load_tokens(const std::filesystem::path & path); + +struct LoadedModel { + std::string path; + ZipVoiceConfig config; + ZipVoiceWeights weights; + ggml_backend_t backend = nullptr; + bool owns_backend = true; // false when the backend was injected by the caller + core::BackendType backend_type = core::BackendType::Cpu; // concrete resolved type + std::mutex mutex; // serializes graph use (graphs are not thread-safe) + runtime::CacheSlots> text_graphs{1}; + struct FmKey { + int64_t t; + int64_t b; + bool guidance; + bool operator==(const FmKey & o) const { return t == o.t && b == o.b && guidance == o.guidance; } + }; + runtime::CacheSlots> fm_graphs{1}; + runtime::CacheSlots, std::unique_ptr> tokenizers{1}; + std::unique_ptr vocos_graph; + std::string vocos_path; + ~LoadedModel() { + vocos_graph.reset(); + text_graphs.clear(); + fm_graphs.clear(); + weights = {}; + if (owns_backend) { + ggml_backend_free(backend); + } + } +}; + +std::vector decode_vocos(LoadedModel & model, const std::string & path, + const std::vector & mel, int threads) { + if (!model.vocos_graph || model.vocos_path != path) { + model.vocos_graph.reset(); + model.vocos_graph = std::make_unique(path, model.backend, threads); + model.vocos_path = path; + } + return model.vocos_graph->decode(mel); +} + +// Sidecar resolution: spec-registered resources (GGUF-embedded sidecars +// materialized by the framework, or files found in the development directory) +// take priority; loose files next to the checkpoint remain the fallback for +// direct API entry points (parity harnesses) that pass no bundle. +struct SidecarResolver { + const engine::assets::ResourceBundle * resources = nullptr; + + std::filesystem::path path(std::string_view id, const std::filesystem::path & dir, + std::string_view filename) const { + if (resources != nullptr) { + if (const auto * p = resources->find_file(id)) return *p; + } + return dir / filename; + } +}; + +// Resolve the requested backend: an injected backend wins; otherwise the +// device's backend type (BestAvailable by default = GPU first, CPU fallback). +core::BackendType resolve_backend_type(const ZipVoiceComputeDevice & device) { + if (device.backend != nullptr) { + return core::backend_type(device.backend); + } + return device.backend_type; +} + +} // namespace + +class ZipVoiceRuntimeState { +public: + std::mutex mutex; + std::shared_ptr model; + std::string key; +}; + +std::shared_ptr make_zipvoice_runtime_state() { + return std::make_shared(); +} + +namespace { +std::shared_ptr load_model( + const std::string & path, const SidecarResolver & sidecars, + const ZipVoiceComputeDevice & device) { + if (!device.runtime) throw std::invalid_argument("zipvoice: runtime is null"); + auto & state = *device.runtime; + std::lock_guard lock(state.mutex); + const auto cache_config_dir = std::filesystem::is_directory(path) + ? std::filesystem::path(path) : std::filesystem::path(path).parent_path(); + const std::string key = path + "::" + sidecars.path("model_config", cache_config_dir, "model.json").string() + + "::" + sidecars.path("tokens", cache_config_dir, "tokens.txt").string() + + "::" + std::to_string(static_cast(resolve_backend_type(device))) + + "::" + std::to_string(device.device_index) + "::" + std::to_string(reinterpret_cast(device.backend)); + if (state.model && state.key == key) return state.model; + state.model.reset(); + auto model = std::make_unique(); + model->path = path; + namespace fs = std::filesystem; + const fs::path p(path); + std::string prefix = ""; + std::shared_ptr source; + if (p.extension() == ".gguf") { + source = engine::assets::make_prefixed_tensor_source( + engine::assets::open_tensor_source(p), "model"); + } else { + // development layout: directory with -orig.safetensors + std::vector candidates; + if (fs::is_regular_file(p) && p.extension() == ".safetensors") { + candidates.push_back(p); + } else if (fs::is_directory(p)) { + for (const auto & entry : fs::directory_iterator(p)) { + if (entry.path().extension() == ".safetensors" && + entry.path().filename().string().find("vocos") == std::string::npos) { + candidates.push_back(entry.path()); + } + } + } + if (candidates.empty()) { + throw std::runtime_error("zipvoice: no safetensors checkpoint in " + path); + } + std::sort(candidates.begin(), candidates.end()); + source = engine::assets::open_tensor_source(candidates.back()); + } + const fs::path config_dir = fs::is_directory(p) ? p : p.parent_path(); + const auto config_path = sidecars.path("model_config", config_dir, "model.json"); + model->config = load_zipvoice_config(config_dir, source.get(), &config_path); + // vocab size from the embedding itself + model->config.vocab_size = static_cast( + source->require_metadata("embed.weight").shape[0]); + // pad id from tokens.txt (the "_" entry per the reference tokenizers) + { + const auto vocab = load_tokens(sidecars.path("tokens", config_dir, "tokens.txt")); + const auto it = vocab.find("_"); + if (it != vocab.end()) { + model->config.pad_id = it->second; + } + } + // Device-selected backend: a caller-injected backend (borrowed, not + // owned) or one created here — CPU compute with n_threads, or a GPU + // backend via BestAvailable (Metal on Apple Silicon, CUDA on NVIDIA). + if (device.backend != nullptr) { + model->backend = device.backend; + model->owns_backend = false; + } else { + core::BackendConfig cfg; + cfg.type = device.backend_type; + cfg.device = device.device_index; + cfg.threads = device.threads > 0 + ? device.threads + : static_cast(std::thread::hardware_concurrency()); + model->backend = core::init_backend(cfg); + model->owns_backend = true; + } + if (model->backend == nullptr) { + throw std::runtime_error("zipvoice: backend init failed"); + } + // BestAvailable resolves to a concrete backend (Metal on Apple + // Silicon); downstream allocation and vocoder decisions use it. + model->backend_type = core::backend_type(model->backend); + model->weights = load_zipvoice_weights( + *source, "", model->config, model->backend, model->backend_type); + + state.key = key; + state.model = std::move(model); + return state.model; +} + +// timestep_embedding host copy (matches zipvoice.modules.timestep_embedding) +std::vector timestep_embedding(float t, int64_t dim) { + const int64_t half = dim / 2; + std::vector out(static_cast(dim)); + for (int64_t i = 0; i < half; ++i) { + const float freq = std::exp(-std::log(10000.0F) * static_cast(i) / + static_cast(half)); + const float arg = t * freq; + out[static_cast(i)] = std::cos(arg); + out[static_cast(half + i)] = std::sin(arg); + } + return out; +} + +void upload_leaf(ggml_backend_t backend, ggml_tensor * leaf, const void * data, size_t bytes) { + (void)backend; + ggml_backend_tensor_set(leaf, data, 0, bytes); // void return; CPU sync path +} + +int compute_threads(const ZipVoiceComputeDevice & device) { + return device.threads > 0 ? device.threads + : static_cast(std::thread::hardware_concurrency()); +} + +// Fill per-stack pad bias / conv gate leaves given the valid frame count in +// the FULL-resolution sequence (frames >= valid are padding). +void fill_masks( + ggml_backend_t backend, + const ZipVoiceConfig & config, + FmDecoderGraph & g, + int64_t valid_frames) { + for (size_t s = 0; s < config.fm_downsampling_factor.size(); ++s) { + const int64_t ds = config.fm_downsampling_factor[s]; + // stack s runs at ceil(T / ds) frames (factors are relative to the + // full frame rate); the ::ds-sampled mask keeps ceil(valid / ds) + // valid frames. + const int64_t T_s = (g.T + ds - 1) / ds; + const int64_t valid_s = std::min((valid_frames + ds - 1) / ds, T_s); + std::vector bias(static_cast(T_s)); + std::vector gate(static_cast(T_s)); + for (int64_t i = 0; i < T_s; ++i) { + const bool pad = i >= valid_s; + bias[static_cast(i)] = pad ? -1000.0F : 0.0F; + gate[static_cast(i)] = pad ? 0.0F : 1.0F; + } + upload_leaf(backend, g.pad_bias[s], bias.data(), bias.size() * sizeof(float)); + upload_leaf(backend, g.conv_gate[s], gate.data(), gate.size() * sizeof(float)); + } +} + +struct VelocityInputs { + // xt, text_condition, speech_condition: [T, F] row-major (t, f) + std::vector xt, text_condition, speech_condition; +}; + +// Run one fm_decoder evaluation. B = 2 duplicates every input (CFG halves +// are prepared by the caller through `halves`). +std::vector run_velocity( + LoadedModel & model, + const ZipVoiceComputeDevice & device, + int64_t T, + int64_t B, + const float * xt, // [B, T, F] + const float * text_cond, // [B, T, F] + const float * speech_cond, // [B, T, F] + int64_t valid_frames, + float t_value, + const float * guidance_embedding, // nullptr for base model + float * scratch_time = nullptr) { + const auto & config = model.config; + const bool with_guidance = guidance_embedding != nullptr; + std::lock_guard lock(model.mutex); + FmDecoderGraph * g = nullptr; + const LoadedModel::FmKey key{T, B, with_guidance}; + if (const auto * slot = model.fm_graphs.find(key)) { + g = slot->get(); + } else { + model.fm_graphs.clear(); + auto built = build_fm_decoder_graph( + model.weights, config, T, B, with_guidance, false, model.backend); + model.fm_graphs.put(key, std::make_unique(std::move(built))); + g = model.fm_graphs.find(key)->get(); + } + + // x_cat [3F, T, B]: interleave channels (xt | text | speech) + const int64_t F = config.feat_dim; + std::vector x_cat(static_cast(3 * F * T * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t i = 0; i < T; ++i) { + float * dst = x_cat.data() + static_cast(3 * F * (i + b * T)); + const float * x_src = xt + static_cast((i + b * T) * F); + const float * t_src = text_cond + static_cast((i + b * T) * F); + const float * s_src = speech_cond + static_cast((i + b * T) * F); + std::memcpy(dst, x_src, F * sizeof(float)); + std::memcpy(dst + F, t_src, F * sizeof(float)); + std::memcpy(dst + 2 * F, s_src, F * sizeof(float)); + } + } + upload_leaf(model.backend, g->x_cat, x_cat.data(), x_cat.size() * sizeof(float)); + const auto time_values = scratch_time != nullptr + ? std::vector(scratch_time, scratch_time + config.time_embed_dim) + : timestep_embedding(t_value, config.time_embed_dim); + upload_leaf(model.backend, g->time_emb, time_values.data(), + static_cast(config.time_embed_dim) * sizeof(float)); + if (with_guidance) { + upload_leaf(model.backend, g->guidance_emb, guidance_embedding, + static_cast(config.time_embed_dim) * sizeof(float)); + } + fill_masks(model.backend, config, *g, valid_frames); + + core::set_backend_threads(model.backend, compute_threads(device)); + const auto status = core::compute_backend_graph(model.backend, g->graph, nullptr, "zipvoice_fm"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("zipvoice: fm decoder graph compute failed"); + } + ggml_backend_synchronize(model.backend); + + // output [F, T, B] -> [B, T, F] + std::vector out_vec(ggml_nelements(g->output)); + ggml_backend_tensor_get(g->output, out_vec.data(), 0, out_vec.size() * sizeof(float)); + const float * out = out_vec.data(); + std::vector v(static_cast(T * F * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t i = 0; i < T; ++i) { + for (int64_t f = 0; f < F; ++f) { + v[static_cast((i + b * T) * F + f)] = + out[static_cast(f + i * F + b * T * F)]; + } + } + } + return v; +} + +// forward_text_inference_ratio_duration (single utterance) +struct TextConditionResult { + std::vector condition; // [T, F] + int64_t T = 0; +}; + +TextConditionResult compute_text_condition( + LoadedModel & model, + const ZipVoiceComputeDevice & device, + const std::vector & tokens, // target tokens + const std::vector & prompt_tokens, // prompt tokens + int64_t prompt_features_len, + float speed) { + // pad_labels appends one trailing pad token; the original token count + // defines the attention and convolution masks. + // NOTE: tokens_lens in the reference EXCLUDES the pad token, so the + // duration split uses token_count, not S. + std::vector cat(prompt_tokens); + cat.insert(cat.end(), tokens.begin(), tokens.end()); + const int64_t token_count = static_cast(cat.size()); + if (!std::isfinite(speed) || speed <= 0 || prompt_features_len <= 0 || tokens.empty()) { + throw std::invalid_argument("zipvoice: nonempty tokens, positive prompt length and speed are required"); + } + for (const int32_t token : cat) { + if (token < 0 || token >= model.config.vocab_size) { + throw std::invalid_argument("zipvoice: token id out of vocabulary range"); + } + } + cat.push_back(model.config.pad_id); + const int64_t S = static_cast(cat.size()); + if (token_count == 0 || prompt_tokens.empty()) { + throw std::runtime_error("zipvoice: empty token stream"); + } + const int64_t features_len = prompt_features_len + + static_cast(std::ceil( + static_cast(prompt_features_len) / + static_cast(prompt_tokens.size()) * + static_cast(tokens.size()) / speed)); + const int64_t T = features_len; + + std::lock_guard lock(model.mutex); + TextEncoderGraph * g = nullptr; + if (const auto * slot = model.text_graphs.find(S)) { + g = slot->get(); + } else { + model.text_graphs.clear(); + auto built = build_text_encoder_graph(model.weights, model.config, S, false, model.backend); + model.text_graphs.put(S, std::make_unique(std::move(built))); + g = model.text_graphs.find(S)->get(); + } + upload_leaf(model.backend, g->token_ids, cat.data(), cat.size() * sizeof(int32_t)); + std::vector zeros(static_cast(S), 0.0F); + std::vector ones(static_cast(S), 1.0F); + zeros.back() = -1000.0F; + ones.back() = 0.0F; + upload_leaf(model.backend, g->pad_bias[0], zeros.data(), zeros.size() * sizeof(float)); + upload_leaf(model.backend, g->conv_gate[0], ones.data(), ones.size() * sizeof(float)); + + core::set_backend_threads(model.backend, compute_threads(device)); + const auto status = core::compute_backend_graph(model.backend, g->graph, nullptr, "zipvoice_text"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("zipvoice: text encoder graph compute failed"); + } + ggml_backend_synchronize(model.backend); + + // embed output [feat_dim, S] -> gather per-frame rows + std::vector embed_vec(ggml_nelements(g->output)); + ggml_backend_tensor_get(g->output, embed_vec.data(), 0, embed_vec.size() * sizeof(float)); + const float * embed = embed_vec.data(); + const int64_t F = model.config.feat_dim; + // prepare_avg_tokens_durations + get_tokens_index + const int64_t avg = T / token_count; // num_frames // num_tokens (B = 1) + std::vector durations(static_cast(token_count), avg); + durations.push_back(T - avg * token_count); + std::vector tokens_index(static_cast(T)); + { + int64_t cur = 0; + for (int64_t i = 0; i < S; ++i) { + const int64_t d = durations[static_cast(i)]; + for (int64_t k = 0; k < d; ++k) { + if (cur + k < T) tokens_index[static_cast(cur + k)] = static_cast(i); + } + cur += d; + } + } + TextConditionResult result; + result.T = T; + result.condition.resize(static_cast(T * F)); + for (int64_t i = 0; i < T; ++i) { + const int32_t tok = tokens_index[static_cast(i)]; + std::memcpy(result.condition.data() + i * F, + embed + static_cast(tok) * F, F * sizeof(float)); + } + return result; +} + +// Euler solver with t_shift; mirrors DiffusionModel/EulerSolver exactly. +std::vector run_sampler( + LoadedModel & model, + const ZipVoiceComputeDevice & device, + const TextConditionResult & text, + const std::vector & speech_condition, // [T, F] padded prompt (zeros elsewhere) + int64_t valid_frames, + const std::vector & x0, + int num_step, + float guidance_scale, + float t_shift) { + const auto & config = model.config; + const int64_t T = text.T; + const int64_t F = config.feat_dim; + const bool distill = config.guidance_scale_embed; + if (num_step <= 0 || num_step > 64 || !std::isfinite(t_shift) || t_shift <= 0 || + !std::isfinite(guidance_scale) || guidance_scale < 0 || + x0.size() != static_cast(T * F)) { + throw std::invalid_argument("zipvoice: invalid sampler parameters or noise shape"); + } + + // timesteps + std::vector ts(num_step + 1); + for (int i = 0; i <= num_step; ++i) { + ts[static_cast(i)] = static_cast(i) / num_step; + } + for (auto & v : ts) { + v = t_shift * v / (1.0F + (t_shift - 1.0F) * v); + } + + std::vector x = x0; + if (distill) { + const auto guidance_embedding = timestep_embedding(guidance_scale, config.time_embed_dim); + for (int step = 0; step < num_step; ++step) { + const float t = ts[static_cast(step)]; + const float dt = ts[static_cast(step + 1)] - t; + auto v = run_velocity(model, device, T, 1, x.data(), text.condition.data(), + speech_condition.data(), valid_frames, t, + guidance_embedding.data()); + for (size_t i = 0; i < x.size(); ++i) { + x[i] += v[i] * dt; + } + } + return x; + } + + // base model: batched CFG + const bool use_cfg = guidance_scale != 0.0F; + if (!use_cfg) { + for (int step = 0; step < num_step; ++step) { + const float t = ts[static_cast(step)]; + const float dt = ts[static_cast(step + 1)] - t; + auto v = run_velocity(model, device, T, 1, x.data(), text.condition.data(), + speech_condition.data(), valid_frames, t, nullptr); + for (size_t i = 0; i < x.size(); ++i) x[i] += v[i] * dt; + } + return x; + } + + float effective = guidance_scale; + std::vector xt2, text2, speech2; + for (int step = 0; step < num_step; ++step) { + const float t = ts[static_cast(step)]; + const float dt = ts[static_cast(step + 1)] - t; + float scale = effective; + xt2.assign(x.data(), x.data() + x.size()); + xt2.insert(xt2.end(), x.begin(), x.end()); + text2.assign(text.condition.size(), 0.0F); + text2.insert(text2.end(), text.condition.begin(), text.condition.end()); + if (t > 0.5F) { + speech2.assign(speech_condition.size(), 0.0F); + speech2.insert(speech2.end(), speech_condition.begin(), speech_condition.end()); + } else { + scale = effective * 2.0F; + speech2.assign(speech_condition.begin(), speech_condition.end()); + speech2.insert(speech2.end(), speech_condition.begin(), speech_condition.end()); + } + auto v = run_velocity(model, device, T, 2, xt2.data(), text2.data(), + speech2.data(), valid_frames, t, nullptr); + std::vector combined(x.size()); + for (size_t i = 0; i < x.size(); ++i) { + const float cond = v[x.size() + i]; + const float uncond = v[i]; + combined[i] = (1.0F + scale) * cond - scale * uncond; + } + for (size_t i = 0; i < x.size(); ++i) x[i] += combined[i] * dt; + } + return x; +} + +struct Rng { + uint64_t state; + explicit Rng(uint64_t seed) : state(seed ? seed : 0x9E3779B97F4A7C15ULL) {} + uint64_t next_u64() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; + } + float next_f32() { + return static_cast(static_cast(next_u64() >> 11) / 9007199254740992.0); + } + float normal() { + const float u1 = std::max(next_f32(), 1e-7F); + const float u2 = next_f32(); + return std::sqrt(-2.0F * std::log(u1)) * std::cos(2.0F * static_cast(kPi) * u2); + } +}; + +// tokens.txt vocab: token -> id (tab separated) +std::unordered_map load_tokens(const std::filesystem::path & tokens_path) { + std::unordered_map map; + const auto & path = tokens_path; + std::ifstream f(path); + if (!f) throw std::runtime_error("zipvoice: cannot open " + path.string()); + std::string line; + while (std::getline(f, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + map.emplace(line.substr(0, tab), std::stoi(line.substr(tab + 1))); + } + if (map.empty()) throw std::runtime_error("zipvoice: empty tokens.txt"); + return map; +} + +std::vector utf8_chars(const std::string & s) { + std::vector out; + for (size_t i = 0; i < s.size();) { + size_t len = 1; + const auto c = static_cast(s[i]); + if (c >= 0xF0) len = 4; + else if (c >= 0xE0) len = 3; + else if (c >= 0xC0) len = 2; + out.emplace_back(s, i, len); + i += len; + } + return out; +} + +} // namespace + +void zipvoice_clear_runtime(const ZipVoiceComputeDevice & device) { + if (!device.runtime) return; + std::lock_guard lock(device.runtime->mutex); + device.runtime->model.reset(); + device.runtime->key.clear(); +} + +std::vector zipvoice_vocos_decode_on_device( + const std::string & model_path, + const std::string & vocos_path, + const std::vector & mel_rows, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + std::lock_guard lock(model.mutex); + return decode_vocos(model, vocos_path, mel_rows, compute_threads(device)); +} + +std::vector zipvoice_text_condition( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + int64_t prompt_features_len, + float speed, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + auto result = compute_text_condition( + model, device, tokens, prompt_tokens, prompt_features_len, speed); + return result.condition; +} + +std::vector zipvoice_text_encoder_raw( + const std::string & model_path, + const std::vector & token_ids, + ZipVoiceLayerTaps * layer_taps, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + std::vector cat(token_ids); + cat.push_back(model.config.pad_id); + const int64_t S = static_cast(cat.size()); + + std::lock_guard lock(model.mutex); + TextEncoderGraph * g = nullptr; + if (const auto * slot = model.text_graphs.find(S)) { + g = slot->get(); + } else { + model.text_graphs.clear(); + auto built = build_text_encoder_graph(model.weights, model.config, S, false, model.backend); + model.text_graphs.put(S, std::make_unique(std::move(built))); + g = model.text_graphs.find(S)->get(); + } + upload_leaf(model.backend, g->token_ids, cat.data(), cat.size() * sizeof(int32_t)); + std::vector zeros(static_cast(S), 0.0F); + std::vector ones(static_cast(S), 1.0F); + zeros.back() = -1000.0F; + ones.back() = 0.0F; + upload_leaf(model.backend, g->pad_bias[0], zeros.data(), zeros.size() * sizeof(float)); + upload_leaf(model.backend, g->conv_gate[0], ones.data(), ones.size() * sizeof(float)); + + core::set_backend_threads(model.backend, compute_threads(device)); + const auto status = core::compute_backend_graph(model.backend, g->graph, nullptr, "zipvoice_text"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("zipvoice: text encoder graph compute failed"); + } + ggml_backend_synchronize(model.backend); + + // output [feat_dim, S] -> [S, feat_dim] rows + std::vector out_vec(ggml_nelements(g->output)); + ggml_backend_tensor_get(g->output, out_vec.data(), 0, out_vec.size() * sizeof(float)); + const int64_t F = model.config.feat_dim; + std::vector rows(static_cast(S * F)); + for (int64_t i = 0; i < S; ++i) { + for (int64_t f = 0; f < F; ++f) { + rows[static_cast(i * F + f)] = + out_vec[static_cast(f + i * F)]; + } + } + if (layer_taps != nullptr) { + layer_taps->layers.clear(); + layer_taps->stages.clear(); + for (ggml_tensor * tap : g->stage_taps) { + // raw flat dump in ggml ne order; python/test side knows each + // tap's shape and reorders + std::vector tap_vec(ggml_nelements(tap)); + ggml_backend_tensor_get(tap, tap_vec.data(), 0, tap_vec.size() * sizeof(float)); + layer_taps->stages.push_back(std::move(tap_vec)); + } + for (ggml_tensor * tap : g->layer_taps) { + const int64_t C = tap->ne[0]; + std::vector tap_vec(ggml_nelements(tap)); + ggml_backend_tensor_get(tap, tap_vec.data(), 0, tap_vec.size() * sizeof(float)); + std::vector tap_rows(static_cast(S * C)); + for (int64_t i = 0; i < S; ++i) { + for (int64_t c = 0; c < C; ++c) { + tap_rows[static_cast(i * C + c)] = + tap_vec[static_cast(c + i * C)]; + } + } + layer_taps->layers.push_back(std::move(tap_rows)); + } + } + return rows; +} + +std::vector zipvoice_velocity( + const std::string & model_path, + const std::vector & xt, + const std::vector & text_condition, + const std::vector & speech_condition, + int64_t features_len, + float t, + float guidance_scale, + const ZipVoiceComputeDevice & device, + int batch_size, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + const int64_t T = features_len; + const float * guidance = nullptr; + std::vector guidance_embedding; + if (model.config.guidance_scale_embed) { + guidance_embedding = timestep_embedding(guidance_scale, model.config.time_embed_dim); + guidance = guidance_embedding.data(); + } + const size_t expected = static_cast(T * model.config.feat_dim * batch_size); + if (T <= 0 || batch_size <= 0 || xt.size() != expected || + text_condition.size() != expected || speech_condition.size() != expected) { + throw std::invalid_argument("zipvoice: velocity input shape mismatch"); + } + return run_velocity(model, device, T, batch_size, xt.data(), text_condition.data(), + speech_condition.data(), features_len, t, guidance); +} + +std::vector zipvoice_sample( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + const std::vector & prompt_features, + int64_t prompt_features_len, + const std::vector & x0, + int num_steps, + float guidance_scale, + float t_shift, + float speed, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + auto text = compute_text_condition( + model, device, tokens, prompt_tokens, prompt_features_len, speed); + const int64_t T = text.T; + const int64_t F = model.config.feat_dim; + if (prompt_features_len <= 0 || prompt_features.size() < static_cast(prompt_features_len * F)) { + throw std::invalid_argument("zipvoice: prompt feature shape mismatch"); + } + std::vector speech(static_cast(T * F), 0.0F); + for (int64_t i = 0; i < prompt_features_len && i < T; ++i) { + std::memcpy(speech.data() + i * F, + prompt_features.data() + static_cast(i * F), F * sizeof(float)); + } + return run_sampler(model, device, text, speech, T, x0, num_steps, guidance_scale, t_shift); +} + +ZipVoiceSynthesisResult zipvoice_synthesize( + const std::string & model_path, + const std::string & vocos_path, + const ZipVoiceSynthesisRequest & request, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + if (request.ref_audio.empty() || request.ref_sample_rate <= 0 || + !std::isfinite(request.feat_scale) || request.feat_scale <= 0) { + throw std::invalid_argument("zipvoice: valid reference audio and positive feature scale are required"); + } + const SidecarResolver sidecars{resources}; + auto owner = load_model(model_path, sidecars, device); + auto & model = *owner; + + // tokens + std::vector tokens = request.token_ids; + std::vector prompt_tokens = request.prompt_token_ids; + if (tokens.empty() || prompt_tokens.empty()) { + // Keep the dictionary frontend with the session, just like the graphs. + // Loading its tables and Jieba dictionary for every chunk is avoidable. + // Serialize encoding too: Emilia lazily initializes its phonemizer. + std::lock_guard lock(model.mutex); + const std::filesystem::path path(model_path); + const std::filesystem::path dir = + std::filesystem::is_directory(path) ? path : path.parent_path(); + const auto vocab = load_tokens(sidecars.path("tokens", dir, "tokens.txt")); + std::unique_ptr phonemizer; + EmiliaTokenizer * emilia = nullptr; + if (request.tokenizer == "espeak") { + phonemizer = std::make_unique( + std::filesystem::path{request.espeak_library_path}, request.espeak_data_path, + std::vector{request.lang}); + } else if (request.tokenizer == "emilia") { + // Chinese/mixed frontend: needs the baked pypinyin tables (from + // the bundle or next to tokens.txt) and an espeak-ng installation + // for English segments. + EmiliaTokenizer::TablePaths tables; + tables.chars = sidecars.path("zh_chars", dir, "zh_chars.tsv"); + tables.phrases = sidecars.path("zh_phrases", dir, "zh_phrases.tsv"); + tables.syllables = sidecars.path("zh_syllables", dir, "zh_syllables.tsv"); + tables.jieba_dict = sidecars.path("zh_jieba_dict", dir, "zh_jieba_dict.txt"); + tables.hmm_model = sidecars.path("zh_hmm_model", dir, "zh_hmm_model.txt"); + const std::vector key{ + sidecars.path("tokens", dir, "tokens.txt").string(), + tables.chars.string(), tables.phrases.string(), tables.syllables.string(), + tables.jieba_dict.string(), tables.hmm_model.string(), + request.espeak_library_path, request.espeak_data_path, request.lang}; + if (!model.tokenizers.find(key)) { + model.tokenizers.clear(); + model.tokenizers.put(key, std::make_unique( + tables, vocab, EmiliaTokenizer::EspeakConfig{ + request.espeak_library_path, request.espeak_data_path, request.lang})); + } + emilia = model.tokenizers.find(key)->get(); + } else if (request.tokenizer != "simple") { + throw std::invalid_argument("zipvoice: tokenizer must be espeak, emilia, or simple"); + } + const auto encode = [&](const std::string & text) { + if (emilia) { + return emilia->encode(text); + } + std::string phones = text; + if (phonemizer) { + phones.clear(); + size_t start = 0; + for (size_t i = 0; i <= text.size(); ++i) { + if (i != text.size() && std::string(";:,.!?\"").find(text[i]) == std::string::npos) continue; + const auto segment = text.substr(start, i - start); + if (!segment.empty()) { + if (!phones.empty() && std::isspace(static_cast(segment.front()))) phones += ' '; + phones += phonemizer->phonemize(segment, 2); + } + if (i != text.size()) phones += text[i]; + start = i + 1; + } + } + std::vector ids; + for (const auto & ch : utf8_chars(phones)) { + if (const auto it = vocab.find(ch); it != vocab.end()) { + ids.push_back(it->second); + } + } + return ids; + }; + if (tokens.empty()) tokens = encode(request.text); + if (prompt_tokens.empty()) prompt_tokens = encode(request.ref_text); + } + + // prompt audio -> 24k mono + auto wav = audio::convert_interleaved_audio_to_mono_torchaudio_sinc_hann_resampled( + request.ref_audio, request.ref_sample_rate, request.ref_channels, kSampleRate); + float prompt_rms = 0.0F; + for (const float s : wav) prompt_rms += s * s; + prompt_rms = std::sqrt(prompt_rms / std::max(1, wav.size())); + if (request.target_rms > 0.0F && prompt_rms < request.target_rms) { + const float gain = request.target_rms / std::max(prompt_rms, 1e-8F); + for (auto & s : wav) s *= gain; + } + + const auto start = std::chrono::steady_clock::now(); + auto mel = zipvoice_logmel(wav); // [T_p, 100] + const int64_t prompt_features_len = static_cast(mel.size()) / kNMel; + const float feat_scale = request.feat_scale; + for (auto & v : mel) v *= feat_scale; + + auto text = compute_text_condition( + model, device, tokens, prompt_tokens, prompt_features_len, request.speed); + const int64_t T = text.T; + const int64_t F = model.config.feat_dim; + + std::vector speech(static_cast(T * F), 0.0F); + std::memcpy(speech.data(), mel.data(), + static_cast(prompt_features_len) * static_cast(F) * sizeof(float)); + + Rng rng(request.seed); + std::vector x0(static_cast(T * F)); + for (auto & v : x0) v = rng.normal(); + + auto x1 = run_sampler( + model, device, text, speech, T, x0, request.num_steps, + request.guidance_scale, request.t_shift); + + // strip prompt frames, unscale features + const int64_t out_frames = T - prompt_features_len; + std::vector features(static_cast(out_frames * F)); + for (int64_t i = 0; i < out_frames; ++i) { + for (int64_t f = 0; f < F; ++f) { + features[static_cast(i * F + f)] = + x1[static_cast((prompt_features_len + i) * F + f)] / feat_scale; + } + } + + std::vector audio; + { + std::lock_guard lock(model.mutex); + audio = decode_vocos(model, vocos_path, features, compute_threads(device)); + } + for (auto & s : audio) s = std::clamp(s, -1.0F, 1.0F); + if (request.target_rms > 0.0F && prompt_rms < request.target_rms && prompt_rms > 0.0F) { + const float gain = prompt_rms / request.target_rms; + for (auto & s : audio) s *= gain; + } + + ZipVoiceSynthesisResult result; + result.sample_rate = kSampleRate; + result.audio = std::move(audio); + const auto end = std::chrono::steady_clock::now(); + result.model_seconds = std::chrono::duration(end - start).count(); + result.audio_seconds = static_cast(result.audio.size()) / kSampleRate; + return result; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/weights.cpp b/src/community_models/zipvoice/weights.cpp new file mode 100644 index 000000000..3844f2ca4 --- /dev/null +++ b/src/community_models/zipvoice/weights.cpp @@ -0,0 +1,257 @@ +#include "engine/community_models/zipvoice/weights.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/io/json.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +std::string read_file(const std::filesystem::path & path) { + std::ifstream stream(path, std::ios::binary); + if (!stream) { + throw std::runtime_error("cannot open " + path.string()); + } + std::ostringstream buffer; + buffer << stream.rdbuf(); + return buffer.str(); +} + +std::vector require_int_list(const io::json::Value & object, const std::string & key) { + const auto & array = object.require(key); + std::vector values; + for (const auto & item : array.as_array()) { + values.push_back(static_cast(item.as_number())); + } + return values; +} + +} // namespace + +ZipVoiceConfig load_zipvoice_config( + const std::filesystem::path & model_dir, + const engine::assets::TensorSource * probe, + const std::filesystem::path * config_path) { + ZipVoiceConfig config; + const auto config_path_resolved = + config_path != nullptr ? *config_path : model_dir / "model.json"; + if (std::filesystem::is_regular_file(config_path_resolved)) { + const auto root = io::json::parse_file(config_path_resolved); + const auto & model = root.require("model"); + config.fm_downsampling_factor = require_int_list(model, "fm_decoder_downsampling_factor"); + config.fm_num_layers = require_int_list(model, "fm_decoder_num_layers"); + config.fm_cnn_kernel = require_int_list(model, "fm_decoder_cnn_module_kernel"); + config.fm_feedforward_dim = io::json::require_i32(model, "fm_decoder_feedforward_dim"); + config.fm_num_heads = io::json::require_i32(model, "fm_decoder_num_heads"); + config.fm_dim = io::json::require_i32(model, "fm_decoder_dim"); + config.text_num_layers = io::json::require_i32(model, "text_encoder_num_layers"); + config.text_feedforward_dim = io::json::require_i32(model, "text_encoder_feedforward_dim"); + config.text_cnn_kernel = io::json::require_i32(model, "text_encoder_cnn_module_kernel"); + config.text_num_heads = io::json::require_i32(model, "text_encoder_num_heads"); + config.text_dim = io::json::require_i32(model, "text_encoder_dim"); + config.query_head_dim = io::json::require_i32(model, "query_head_dim"); + config.value_head_dim = io::json::require_i32(model, "value_head_dim"); + config.pos_head_dim = io::json::require_i32(model, "pos_head_dim"); + config.pos_dim = io::json::require_i32(model, "pos_dim"); + config.time_embed_dim = io::json::require_i32(model, "time_embed_dim"); + config.text_embed_dim = io::json::require_i32(model, "text_embed_dim"); + config.feat_dim = io::json::require_i32(model, "feat_dim"); + const auto * feature = root.find("feature"); + if (feature != nullptr) { + config.sampling_rate = io::json::require_i32(*feature, "sampling_rate"); + } + } else { + // Defaults matching the released zipvoice / zipvoice_distill config. + } + if (config.fm_downsampling_factor.size() != config.fm_num_layers.size() || + config.fm_downsampling_factor.size() != config.fm_cnn_kernel.size()) { + throw std::runtime_error("zipvoice: stack config length mismatch in model.json"); + } + if (probe != nullptr) { + config.guidance_scale_embed = probe->has_tensor("fm_decoder.guidance_scale_embed.weight"); + } + return config; +} + +ZipVoiceWeights load_zipvoice_weights( + const engine::assets::TensorSource & raw_source, + const std::string & prefix, + const ZipVoiceConfig & config, + ggml_backend_t backend, + core::BackendType backend_type) { + std::shared_ptr source = + engine::assets::make_prefixed_tensor_source( + std::shared_ptr( + std::shared_ptr(), &raw_source), + prefix); + + ZipVoiceWeights weights; + weights.store = std::make_shared( + backend, backend_type, "zipvoice.weights", 2ULL * 1024ULL * 1024ULL * 1024ULL); + + // Dimension arguments are PHYSICAL ggml order (ne0 fastest). -1 accepts + // any value; non-wildcard entries are validated against the checkpoint + // metadata (which stores torch order, reversed here). + const auto t2 = [&](const std::string & name, int64_t d0, int64_t d1) { + const auto meta = source->require_metadata(name); + if (meta.shape.size() != 2) { + throw std::runtime_error("zipvoice: " + name + " must be rank 2"); + } + if ((d0 >= 0 && d0 != meta.shape[1]) || (d1 >= 0 && d1 != meta.shape[0])) { + throw std::runtime_error("zipvoice: " + name + " shape mismatch"); + } + // Linear and embedding weights can retain their GGUF quantization. + return weights.store->load_tensor( + *source, name, engine::assets::TensorStorageType::Native, meta.shape); + }; + const auto t1 = [&](const std::string & name, int64_t d0) { + const auto meta = source->require_metadata(name); + if (meta.shape.size() != 1) { + throw std::runtime_error("zipvoice: " + name + " must be rank 1"); + } + if (d0 >= 0 && d0 != meta.shape[0]) { + throw std::runtime_error("zipvoice: " + name + " shape mismatch"); + } + return weights.store->load_f32_tensor(*source, name, meta.shape); + }; + const auto t3 = [&](const std::string & name, int64_t d0, int64_t d1, int64_t d2) { + const auto meta = source->require_metadata(name); + if (meta.shape.size() != 3) { + throw std::runtime_error("zipvoice: " + name + " must be rank 3"); + } + if ((d0 >= 0 && d0 != meta.shape[2]) || (d1 >= 0 && d1 != meta.shape[1]) || + (d2 >= 0 && d2 != meta.shape[0])) { + throw std::runtime_error("zipvoice: " + name + " shape mismatch"); + } + return weights.store->load_f32_tensor(*source, name, meta.shape); + }; + + const auto load_zipformer = [&](TTSZipformerWeights & w, const std::string & base, + int encoder_dim, int feedforward_dim, + const std::vector & downsampling, + const std::vector & num_layers, + const std::vector & kernels, + int num_heads, bool with_time) { + w.in_proj_w = t2(base + ".in_proj.weight", -1, encoder_dim); + w.in_proj_b = t1(base + ".in_proj.bias", encoder_dim); + w.out_proj_w = t2(base + ".out_proj.weight", -1, -1); + w.out_proj_b = t1(base + ".out_proj.bias", -1); + if (with_time) { + w.time_mlp0_w = t2(base + ".time_embed.0.weight", config.time_embed_dim, config.time_embed_dim * 2); + w.time_mlp0_b = t1(base + ".time_embed.0.bias", config.time_embed_dim * 2); + w.time_mlp2_w = t2(base + ".time_embed.2.weight", config.time_embed_dim * 2, config.time_embed_dim); + w.time_mlp2_b = t1(base + ".time_embed.2.bias", config.time_embed_dim); + if (config.guidance_scale_embed) { + w.guidance_embed_w = t2(base + ".guidance_scale_embed.weight", config.time_embed_dim, config.time_embed_dim); + } + } + const size_t num_stacks = downsampling.size(); + w.stacks.resize(num_stacks); + for (size_t s = 0; s < num_stacks; ++s) { + auto & stack = w.stacks[s]; + const std::string sp = base + ".encoders." + std::to_string(s); + stack.layers.resize(static_cast(num_layers[s])); + for (size_t l = 0; l < stack.layers.size(); ++l) { + auto & layer = stack.layers[l]; + const std::string lp = sp + (downsampling[s] > 1 ? ".encoder" : "") + + ".layers." + std::to_string(l); + layer.bypass_scale = t1(lp + ".bypass.bypass_scale", encoder_dim); + layer.bypass_mid_scale = t1(lp + ".bypass_mid.bypass_scale", encoder_dim); + layer.attn_in_proj_w = t2(lp + ".self_attn_weights.in_proj.weight", encoder_dim, -1); + layer.attn_in_proj_b = t1(lp + ".self_attn_weights.in_proj.bias", -1); + layer.linear_pos_w = t2(lp + ".self_attn_weights.linear_pos.weight", + config.pos_dim, num_heads * config.pos_head_dim); + layer.sa1_in_w = t2(lp + ".self_attn1.in_proj.weight", encoder_dim, num_heads * config.value_head_dim); + layer.sa1_in_b = t1(lp + ".self_attn1.in_proj.bias", num_heads * config.value_head_dim); + layer.sa1_out_w = t2(lp + ".self_attn1.out_proj.weight", num_heads * config.value_head_dim, encoder_dim); + layer.sa1_out_b = t1(lp + ".self_attn1.out_proj.bias", encoder_dim); + layer.sa2_in_w = t2(lp + ".self_attn2.in_proj.weight", encoder_dim, num_heads * config.value_head_dim); + layer.sa2_in_b = t1(lp + ".self_attn2.in_proj.bias", num_heads * config.value_head_dim); + layer.sa2_out_w = t2(lp + ".self_attn2.out_proj.weight", num_heads * config.value_head_dim, encoder_dim); + layer.sa2_out_b = t1(lp + ".self_attn2.out_proj.bias", encoder_dim); + const int ff1_hidden = feedforward_dim * 3 / 4; + const int ff3_hidden = feedforward_dim * 5 / 4; + layer.ff1_in_w = t2(lp + ".feed_forward1.in_proj.weight", encoder_dim, ff1_hidden); + layer.ff1_in_b = t1(lp + ".feed_forward1.in_proj.bias", ff1_hidden); + layer.ff1_out_w = t2(lp + ".feed_forward1.out_proj.weight", ff1_hidden, encoder_dim); + layer.ff1_out_b = t1(lp + ".feed_forward1.out_proj.bias", encoder_dim); + layer.ff2_in_w = t2(lp + ".feed_forward2.in_proj.weight", encoder_dim, feedforward_dim); + layer.ff2_in_b = t1(lp + ".feed_forward2.in_proj.bias", feedforward_dim); + layer.ff2_out_w = t2(lp + ".feed_forward2.out_proj.weight", feedforward_dim, encoder_dim); + layer.ff2_out_b = t1(lp + ".feed_forward2.out_proj.bias", encoder_dim); + layer.ff3_in_w = t2(lp + ".feed_forward3.in_proj.weight", encoder_dim, ff3_hidden); + layer.ff3_in_b = t1(lp + ".feed_forward3.in_proj.bias", ff3_hidden); + layer.ff3_out_w = t2(lp + ".feed_forward3.out_proj.weight", ff3_hidden, encoder_dim); + layer.ff3_out_b = t1(lp + ".feed_forward3.out_proj.bias", encoder_dim); + const int na_hidden = encoder_dim * 3 / 4; + layer.na_in_w = t2(lp + ".nonlin_attention.in_proj.weight", encoder_dim, na_hidden * 3); + layer.na_in_b = t1(lp + ".nonlin_attention.in_proj.bias", na_hidden * 3); + layer.na_out_w = t2(lp + ".nonlin_attention.out_proj.weight", na_hidden, encoder_dim); + layer.na_out_b = t1(lp + ".nonlin_attention.out_proj.bias", encoder_dim); + const int kernel = kernels[s]; + for (int m = 1; m <= 2; ++m) { + const std::string mp = lp + ".conv_module" + std::to_string(m); + core::TensorValue &in_w = m == 1 ? layer.cm1_in_w : layer.cm2_in_w; + core::TensorValue &in_b = m == 1 ? layer.cm1_in_b : layer.cm2_in_b; + core::TensorValue &cv_w = m == 1 ? layer.cm1_conv_w : layer.cm2_conv_w; + core::TensorValue &cv_b = m == 1 ? layer.cm1_conv_b : layer.cm2_conv_b; + core::TensorValue &out_w = m == 1 ? layer.cm1_out_w : layer.cm2_out_w; + core::TensorValue &out_b = m == 1 ? layer.cm1_out_b : layer.cm2_out_b; + in_w = t2(mp + ".in_proj.weight", encoder_dim, encoder_dim * 2); + in_b = t1(mp + ".in_proj.bias", encoder_dim * 2); + cv_w = t3(mp + ".depthwise_conv.weight", kernel, 1, encoder_dim); + cv_b = t1(mp + ".depthwise_conv.bias", encoder_dim); + out_w = t2(mp + ".out_proj.weight", encoder_dim, encoder_dim); + out_b = t1(mp + ".out_proj.bias", encoder_dim); + } + layer.norm_bias = t1(lp + ".norm.bias", encoder_dim); + const auto log_scale = source->require_f32(lp + ".norm.log_scale"); + if (log_scale.size() != 1) { + throw std::runtime_error("zipvoice: " + lp + ".norm.log_scale must be a scalar"); + } + layer.norm_log_scale = log_scale[0]; + } + if (with_time) { + stack.time_proj_w = t2(sp + (downsampling[s] > 1 ? ".encoder" : "") + ".time_emb.1.weight", + config.time_embed_dim, encoder_dim); + stack.time_proj_b = t1(sp + (downsampling[s] > 1 ? ".encoder" : "") + ".time_emb.1.bias", encoder_dim); + } + if (downsampling[s] > 1) { + const auto bias = source->require_f32( + sp + ".downsample.bias", + std::vector{downsampling[s]}); + stack.downsample_bias = bias; + const float maximum = *std::max_element(bias.begin(), bias.end()); + float sum = 0.0F; + for (float & value : stack.downsample_bias) { + value = std::exp(value - maximum); + sum += value; + } + for (float & value : stack.downsample_bias) value /= sum; + stack.out_combiner_scale = t1(sp + ".out_combiner.bypass_scale", encoder_dim); + } + } + }; + + load_zipformer(weights.fm_decoder, "fm_decoder", config.fm_dim, config.fm_feedforward_dim, + config.fm_downsampling_factor, config.fm_num_layers, config.fm_cnn_kernel, + config.fm_num_heads, true); + load_zipformer(weights.text_encoder, "text_encoder", config.text_dim, config.text_feedforward_dim, + {1}, {config.text_num_layers}, {config.text_cnn_kernel}, + config.text_num_heads, false); + weights.text_encoder.embed_w = t2("embed.weight", config.text_embed_dim, config.vocab_size); + + weights.store->upload(); + source->release_storage(); + return weights; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/zipformer.cpp b/src/community_models/zipvoice/zipformer.cpp new file mode 100644 index 000000000..e2c24ac11 --- /dev/null +++ b/src/community_models/zipvoice/zipformer.cpp @@ -0,0 +1,931 @@ +// TTSZipformer graph construction for the ZipVoice community port. +// +// Reference: zipvoice/models/modules/zipformer.py (k2-fsa/ZipVoice, +// Apache-2.0) evaluated in inference mode. Balancer / Whiten / Identity / +// dropout modules are no-ops; BypassModule reduces to a per-channel learned +// residual scale; ScaledLinear checkpoints already contain folded scales. +// +// Physical layouts (ne0 fastest): +// activations: [C (ne0), T (ne1), B (ne2)] +// attention: [src (ne0), tgt (ne1), H (ne2), B (ne3)] +// Softmax always runs over ne0 (keys), matching torch softmax(dim=-1) on +// (H, B, tgt, src). +// +// The compact relative-position attention lowers torch's overlapping +// as_strided view into an explicit strided ggml view over the contiguous +// relative scores R [2T-1 (ne0), T (ne1), H, B]: +// abs[s, t] = R_flat[(T - 1) + t * (2T - 2) + s] +// where R_flat[m + t * (2T - 1)] = p_t . w_m. Validated against the +// moonlight fixture (tests/zipvoice). + +#include "engine/community_models/zipvoice/zipformer.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/lookup_modules.h" + +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +ZipVoiceGraphResources::~ZipVoiceGraphResources() { + if (backend && graph) core::release_backend_graph_resources(backend, graph, true); + if (gallocr) ggml_gallocr_free(static_cast(gallocr)); + if (buffer) ggml_backend_buffer_free(buffer); + if (ctx) ggml_free(ctx); + if (tensor_ctx) ggml_free(tensor_ctx); +} + +namespace { + +constexpr float kPadBias = -1000.0F; + +// --------------------------------------------------------------------------- +// small helpers (raw ggml, channel-fastest layout) + +// Physical views remain explicit for Zipformer's overlapping relative shift; +// ordinary projections, activation, normalization and convolution use modules. +struct GraphBuilder : core::ModuleBuildContext { + GraphBuilder(ggml_context * c, core::BackendType type) { ggml = c; backend_type = type; } + operator ggml_context *() const { return ggml; } +}; +core::TensorValue value(ggml_tensor * t) { + return core::wrap_tensor(t, core::TensorShape::from_dims({t->ne[3], t->ne[2], t->ne[1], t->ne[0]}), t->type); +} +ggml_tensor * t_cont(GraphBuilder & ctx, ggml_tensor * t) { + return core::ensure_backend_addressable_layout(ctx, value(t)).tensor; +} +ggml_tensor * t_permute(GraphBuilder & ctx, ggml_tensor * t, int p0, int p1, int p2, int p3) { + const int physical[4] = {p0, p1, p2, p3}; + std::array axes; + for (int source = 0; source < 4; ++source) axes[3 - physical[source]] = 3 - source; + return modules::TransposeModule({axes, 4}).build(ctx, value(t)).tensor; +} +ggml_tensor * t_reshape_2d(GraphBuilder & ctx, ggml_tensor * t, int64_t n0, int64_t n1) { + return core::reshape_tensor(ctx, value(t), core::TensorShape::from_dims({n1, n0})).tensor; +} +ggml_tensor * t_reshape_3d(GraphBuilder & ctx, ggml_tensor * t, int64_t n0, int64_t n1, int64_t n2) { + return core::reshape_tensor(ctx, value(t), core::TensorShape::from_dims({n2, n1, n0})).tensor; +} +ggml_tensor * t_reshape_4d(GraphBuilder & ctx, ggml_tensor * t, int64_t n0, int64_t n1, int64_t n2, int64_t n3) { + return core::reshape_tensor(ctx, value(t), core::TensorShape::from_dims({n3, n2, n1, n0})).tensor; +} +ggml_tensor * t_rows(GraphBuilder & ctx, ggml_tensor * table, ggml_tensor * indices) { + return modules::EmbeddingModule({table->ne[1], table->ne[0]}).build(ctx, + core::wrap_tensor(indices, core::TensorShape::from_dims({indices->ne[0]}), GGML_TYPE_I32), + core::wrap_tensor(table, core::TensorShape::from_dims({table->ne[1], table->ne[0]}), table->type)).tensor; +} +ggml_tensor * t_matmul(GraphBuilder & ctx, ggml_tensor * a, ggml_tensor * b) { + if (a->ne[2] != b->ne[2] || a->ne[3] != b->ne[3]) { + // MatMulModule requires equal batch dimensions. Relative-position + // attention and weighted downsampling rely on ggml's batch broadcast. + return ggml_mul_mat(ctx, t_cont(ctx, a), t_cont(ctx, b)); + } + const auto rhs = modules::TransposeModule({{0, 1, 3, 2}, 4}).build(ctx, value(a)); + return modules::MatMulModule().build(ctx, value(b), rhs).tensor; +} +ggml_tensor * t_add(GraphBuilder & ctx, ggml_tensor * a, ggml_tensor * b) { + auto rhs = modules::RepeatModule({value(a).shape}).build(ctx, value(b)); + return modules::AddModule().build(ctx, value(a), rhs).tensor; +} +ggml_tensor * t_mul(GraphBuilder & ctx, ggml_tensor * a, ggml_tensor * b) { + auto rhs = modules::RepeatModule({value(a).shape}).build(ctx, value(b)); + return modules::MulModule().build(ctx, value(a), rhs).tensor; +} +ggml_tensor * t_linear(GraphBuilder & ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) { + modules::LinearWeights weights{core::wrap_tensor(w, core::TensorShape::from_dims({w->ne[1], w->ne[0]}), w->type), std::nullopt}; + if (b) weights.bias = core::wrap_tensor(b, core::TensorShape::from_dims({b->ne[0]}), b->type); + return modules::LinearModule({w->ne[0], w->ne[1], b != nullptr}).build(ctx, value(x), weights).tensor; +} +ggml_tensor * swoosh_l(GraphBuilder & ctx, ggml_tensor * x) { + return modules::SwooshLModule().build(ctx, value(x)).tensor; +} +ggml_tensor * swoosh_r(GraphBuilder & ctx, ggml_tensor * x) { + return modules::SwooshRModule().build(ctx, value(x)).tensor; +} +ggml_tensor * bias_norm(GraphBuilder & ctx, ggml_tensor * x, ggml_tensor * bias, float log_scale) { + return modules::BiasNormModule({x->ne[0]}).build(ctx, value(x), { + core::wrap_tensor(bias, core::TensorShape::from_dims({bias->ne[0]})), log_scale}).tensor; +} + +std::vector compact_relative_position(int64_t seq, int64_t pos_dim) { + constexpr float pi = 3.14159265358979323846F; + const int64_t length = 2 * seq - 1; + std::vector table(static_cast(length * pos_dim), 0.0F); + const float compression = std::sqrt(static_cast(pos_dim)); + const float length_scale = static_cast(pos_dim) / (2.0F * pi); + for (int64_t row = 0; row < length; ++row) { + const float x = static_cast(row - (seq - 1)); + const float sign = x < 0.0F ? -1.0F : (x > 0.0F ? 1.0F : 0.0F); + const float compressed = compression * sign * + (std::log(std::fabs(x) + compression) - std::log(compression)); + const float angle = std::atan(compressed / length_scale); + for (int64_t dim = 0; dim < pos_dim / 2; ++dim) { + const float frequency = static_cast(dim + 1); + table[static_cast(row * pos_dim + 2 * dim)] = std::cos(angle * frequency); + table[static_cast(row * pos_dim + 2 * dim + 1)] = std::sin(angle * frequency); + } + table[static_cast(row * pos_dim + pos_dim - 1)] = 1.0F; + } + return table; +} + +// Host constant factory. On CPU (inline ggml ctx) values are written +// directly; on CUDA (no_alloc ctx) they are staged and uploaded later. +struct StagedConst { + ggml_tensor * tensor; + std::vector bytes; +}; + +struct Consts { + // Tensors are created in the dedicated tensor context (never the graph + // context) so ggml_backend_alloc_ctx_tensors can give them a private + // backend buffer that the graph arena never aliases. + ggml_context * ctx = nullptr; + std::vector * staged = nullptr; + int index = 0; + + template + ggml_tensor * make(const T * data, size_t count, ggml_type type, + std::initializer_list dims) { + std::vector d(dims); + ggml_tensor * t = nullptr; + switch (d.size()) { + case 1: t = ggml_new_tensor_1d(ctx, type, d[0]); break; + case 2: t = ggml_new_tensor_2d(ctx, type, d[0], d[1]); break; + case 3: t = ggml_new_tensor_3d(ctx, type, d[0], d[1], d[2]); break; + case 4: t = ggml_new_tensor_4d(ctx, type, d[0], d[1], d[2], d[3]); break; + default: throw std::runtime_error("zipvoice: bad const rank"); + } + ggml_set_name(t, ("zv_c" + std::to_string(index++)).c_str()); + const size_t bytes = count * sizeof(T); + if (t->data != nullptr) { + std::memcpy(t->data, data, bytes); + } else if (staged != nullptr) { + const auto * raw = reinterpret_cast(data); + staged->push_back({t, std::vector(raw, raw + bytes)}); + } else { + throw std::runtime_error("zipvoice: no home for graph constant"); + } + return t; + } + + ggml_tensor * f32(std::vector values, std::initializer_list dims) { + return make(values.data(), values.size(), GGML_TYPE_F32, dims); + } + ggml_tensor * i32(std::vector values, std::initializer_list dims) { + return make(values.data(), values.size(), GGML_TYPE_I32, dims); + } +}; + +// --------------------------------------------------------------------------- +// attention pieces + +// RelPositionMultiheadAttentionWeights (inference path). +// src [C, T, B] -> weights [src(ne0), tgt(ne1), H(ne2), B(ne3)]. +ggml_tensor * attention_weights( + GraphBuilder & ctx, + Consts & consts, + const ZipLayerWeights & w, + ggml_tensor * src, + ggml_tensor * attn_bias, // [T, 1, 1, 1] additive + int64_t H, + int64_t qh, + int64_t ph, + int64_t pos_dim, + int64_t T, + int64_t B, + ggml_tensor ** tap_qtb = nullptr, + ggml_tensor ** tap_ktb = nullptr, + ggml_tensor ** tap_inproj = nullptr, + ggml_tensor ** tap_scores = nullptr, + ggml_tensor ** tap_pos = nullptr) { + const int64_t q_total = H * qh; + const int64_t M = 2 * T - 1; + + auto * projected = t_linear(ctx, src, w.attn_in_proj_w.tensor, w.attn_in_proj_b.tensor); + if (tap_inproj != nullptr) *tap_inproj = projected; + // [C_total, T, B]: slice channels, reshape (d, H), permute to (d, T, H, B) + // slice [H*per] channels at `offset` (per-head split, head fastest after + // the per-channel dim) -> [per, T, H, B] + const auto qktb = [&](int64_t offset, int64_t total, int64_t per) { + (void)total; + // channel slice as a 4D view [per, H, T, B]: strides for T/B come + // from the SOURCE (frame stride = full channel width), reshape would + // assert on this non-contiguous layout + auto * four = ggml_view_4d(ctx, projected, per, H, T, B, + per * sizeof(float), projected->nb[1], + projected->nb[2], offset * sizeof(float)); + return t_cont(ctx, t_permute(ctx, four, 0, 2, 1, 3)); // [per, T, H, B] + }; + auto * q_tb = qktb(0, q_total, qh); + auto * k_tb = qktb(q_total, q_total, qh); + auto * p_tb = qktb(2 * q_total, H * ph, ph); + if (tap_qtb != nullptr) *tap_qtb = q_tb; + if (tap_ktb != nullptr) *tap_ktb = k_tb; + + // Raw mul_mat is retained for batched contractions: MatMulModule + // requires equal batch dimensions and cannot express the position broadcast. + // scores[src, tgt, H, B] = k^T q + auto * scores = t_matmul(ctx, k_tb, q_tb); + if (tap_scores != nullptr) *tap_scores = scores; + + // relative position scores: R[m, t] = p_t . w_m as [M, T, H, B] + // (mul_mat broadcasts the degenerate batch dims of the pos operand) + auto * pos_emb = consts.f32(compact_relative_position(T, pos_dim), {pos_dim, M}); + auto * pos_proj = t_matmul(ctx, w.linear_pos_w.tensor, pos_emb); // [H*ph, M] + auto * pos_4d = t_reshape_4d(ctx, pos_proj, ph, H, M, 1); + auto * pos_tb = t_cont(ctx, t_permute(ctx, pos_4d, 0, 2, 1, 3)); // [ph, M, H, 1] + auto * rel_t = t_matmul(ctx, pos_tb, p_tb); // [M, T, H, B] contiguous + + // overlapping strided view (torch as_strided lowering): + // abs[s, t] = flat[(T - 1) + t * (2T - 2) + s] + const size_t nb0 = sizeof(float); + const size_t nb1 = static_cast(2 * T - 2) * nb0; + const size_t nb2 = static_cast(M) * static_cast(T) * nb0; + const size_t nb3 = nb2 * static_cast(H); + auto * pos_abs = ggml_view_4d(ctx, rel_t, T, T, H, B, nb1, nb2, nb3, + static_cast(T - 1) * nb0); + auto * pos_final = t_cont(ctx, pos_abs); // [src, tgt, H, B] + + auto * with_pos = t_add(ctx, scores, pos_final); + if (tap_pos != nullptr) *tap_pos = with_pos; + auto * biased = t_add(ctx, with_pos, attn_bias); + return modules::SoftmaxModule().build(ctx, value(biased)).tensor; +} + +// SelfAttention (value path). weights [src, tgt, H, B]; x [C, T, B]. +ggml_tensor * self_attention( + GraphBuilder & ctx, + const ZipLayerWeights & w, + int which, // 1 or 2 + ggml_tensor * x, + ggml_tensor * attn_weights, + int64_t H, + int64_t vh, + int64_t T, + int64_t B) { + const auto & in_w = which == 1 ? w.sa1_in_w : w.sa2_in_w; + const auto & in_b = which == 1 ? w.sa1_in_b : w.sa2_in_b; + const auto & out_w = which == 1 ? w.sa1_out_w : w.sa2_out_w; + const auto & out_b = which == 1 ? w.sa1_out_b : w.sa2_out_b; + + auto * v = t_linear(ctx, x, in_w.tensor, in_b.tensor); // [H*vh, T, B] + auto * v4 = t_reshape_4d(ctx, v, vh, H, T, B); + // [T(src), vh, H, B] for the contraction over src + auto * v_tb = t_cont(ctx, t_permute(ctx, v4, 1, 2, 0, 3)); + auto * attended = t_matmul(ctx, attn_weights, v_tb); // [tgt, vh, H, B] + // back to channel-fastest [H*vh, T, B] + auto * ch = t_cont(ctx, t_permute(ctx, attended, 2, 0, 1, 3)); // [vh, H, T, B] + auto * ch2 = t_reshape_2d(ctx, ch, H * vh, T * B); + auto * out = t_linear(ctx, ch2, out_w.tensor, out_b.tensor); // [C, T*B] + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// NonlinAttention (uses head-0 weights only). +ggml_tensor * nonlin_attention( + GraphBuilder & ctx, + const ZipLayerWeights & w, + ggml_tensor * x, + ggml_tensor * attn_weights, // [src, tgt, H, B] + int64_t C, + int64_t H, + int64_t T, + int64_t B, + ggml_tensor ** tap_gated = nullptr, + ggml_tensor ** tap_attended = nullptr, + ggml_tensor ** tap_gtb = nullptr, + ggml_tensor ** tap_attnrep = nullptr, + ggml_tensor ** tap_mul = nullptr, + ggml_tensor ** tap_h = nullptr, + ggml_tensor ** tap_y2 = nullptr) { + const int64_t hidden = C * 3 / 4; + auto * h = t_linear(ctx, x, w.na_in_w.tensor, w.na_in_b.tensor); // [3*hidden, T, B] + if (tap_h != nullptr) *tap_h = h; + const size_t frame_stride = h->nb[1]; // 3*hidden floats per frame + const size_t batch_stride = h->nb[2]; + auto * s = ggml_view_3d(ctx, h, hidden, T, B, + frame_stride, batch_stride, 0); + auto * xx = ggml_view_3d(ctx, h, hidden, T, B, + frame_stride, batch_stride, hidden * sizeof(float)); + auto * y = ggml_view_3d(ctx, h, hidden, T, B, + frame_stride, batch_stride, 2 * hidden * sizeof(float)); + auto * gated = t_mul(ctx, xx, modules::TanhModule().build(ctx, value(s)).tensor); // [hidden, T, B] + if (tap_gated != nullptr) *tap_gated = gated; + + // All heads share head-0 weights, so the per-head matmul collapses to a + // single 2D contraction: out[t, c] = sum_s attn0[s, t] * gatedT[s, c], + // with gated channels already in torch (h*hd + d) order. + // out[t, c] = sum_s attn0[s, t] * gated_T[s, c]; gated channels are + // already in torch (h*hd + d) order. + auto * attn0_2d = t_cont(ctx, ggml_view_3d(ctx, attn_weights, T, T, B, + attn_weights->nb[1], attn_weights->nb[3], 0)); + if (tap_attnrep != nullptr) *tap_attnrep = attn0_2d; + ggml_tensor * gated_T; + if (B == 1) { + gated_T = t_cont(ctx, t_permute(ctx, gated, 1, 0, 2, 3)); // [s, c] + } else { + auto * g3 = t_reshape_4d(ctx, gated, hidden, T, B, 1); + gated_T = t_cont(ctx, t_permute(ctx, g3, 1, 0, 2, 3)); + } + if (tap_gtb != nullptr) *tap_gtb = gated_T; + auto * attended = t_matmul(ctx, attn0_2d, gated_T); // [t (ne0), c (ne1)] + if (tap_attended != nullptr) *tap_attended = attended; + ggml_tensor * ch2; + if (B == 1) { + ch2 = t_cont(ctx, t_permute(ctx, attended, 1, 0, 2, 3)); // [c, t] + } else { + auto * a3 = t_reshape_4d(ctx, attended, attended->ne[0], attended->ne[1], B, 1); + ch2 = t_cont(ctx, t_permute(ctx, a3, 1, 0, 2, 3)); + } + // y is a strided view; reshape would assert. Slice h directly as 2D: + // element (c, r) with r = t + b*T sits at 2*hidden + c + r*3*hidden. + auto * y2 = t_cont(ctx, ggml_view_2d(ctx, h, hidden, T * B, + h->nb[1], 2 * hidden * sizeof(float))); + if (tap_y2 != nullptr) *tap_y2 = y2; + auto * na_mul = t_mul(ctx, t_reshape_2d(ctx, ch2, hidden, T * B), y2); + if (tap_mul != nullptr) *tap_mul = na_mul; + auto * out = t_linear(ctx, na_mul, w.na_out_w.tensor, w.na_out_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// ConvolutionModule (non-causal, symmetric padding). conv_gate zeroes +// padded frames before the depthwise convolution (torch masked_fill). +ggml_tensor * conv_module( + GraphBuilder & ctx, + const ZipLayerWeights & w, + int which, + ggml_tensor * x, // [C, T, B] + ggml_tensor * conv_gate, // [1, T, 1] (0/1) + int64_t C, + int64_t T, + int64_t B) { + const auto & in_w = which == 1 ? w.cm1_in_w : w.cm2_in_w; + const auto & in_b = which == 1 ? w.cm1_in_b : w.cm2_in_b; + const auto & conv_w = which == 1 ? w.cm1_conv_w : w.cm2_conv_w; + const auto & conv_b = which == 1 ? w.cm1_conv_b : w.cm2_conv_b; + const auto & out_w = which == 1 ? w.cm1_out_w : w.cm2_out_w; + const auto & out_b = which == 1 ? w.cm1_out_b : w.cm2_out_b; + const int64_t kernel = conv_w.tensor->ne[0]; + + auto * h = t_linear(ctx, x, in_w.tensor, in_b.tensor); // [2C, T, B] + const size_t conv_frame_stride = h->nb[1]; // 2C floats per frame + const size_t conv_batch_stride = h->nb[2]; + auto * xv = ggml_view_3d(ctx, h, C, T, B, + conv_frame_stride, conv_batch_stride, 0); + auto * sv = ggml_view_3d(ctx, h, C, T, B, + conv_frame_stride, conv_batch_stride, + C * sizeof(float)); + auto * gated = t_mul(ctx, xv, modules::SigmoidModule().build(ctx, value(sv)).tensor); + gated = t_mul(ctx, gated, conv_gate); // zero padded frames + + auto input = core::wrap_tensor(gated, core::TensorShape::from_dims({B, T, C})); + input = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, input); + auto conv = modules::DepthwiseConv1dModule({C, kernel, 1, int(kernel / 2), 1, true}) + .build(ctx, input, {conv_w, conv_b}); + conv = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, conv); + auto * with_bias = core::ensure_backend_addressable_layout(ctx, conv).tensor; + + // out_proj = SwooshR + linear + auto * activated = swoosh_r(ctx, with_bias); + auto * out = t_linear(ctx, activated, out_w.tensor, out_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// FeedforwardModule: linear -> SwooshL -> linear. +ggml_tensor * feedforward( + GraphBuilder & ctx, + const ZipLayerWeights & w, + int which, + ggml_tensor * x, + int64_t T, + int64_t B, + ggml_tensor ** inproj_out = nullptr, + ggml_tensor ** activated_out = nullptr) { + const auto & in_w = which == 1 ? w.ff1_in_w : which == 2 ? w.ff2_in_w : w.ff3_in_w; + const auto & in_b = which == 1 ? w.ff1_in_b : which == 2 ? w.ff2_in_b : w.ff3_in_b; + const auto & out_w = which == 1 ? w.ff1_out_w : which == 2 ? w.ff2_out_w : w.ff3_out_w; + const auto & out_b = which == 1 ? w.ff1_out_b : which == 2 ? w.ff2_out_b : w.ff3_out_b; + auto * h = t_linear(ctx, x, in_w.tensor, in_b.tensor); + if (inproj_out != nullptr) *inproj_out = h; + auto * activated = swoosh_l(ctx, h); + if (activated_out != nullptr) *activated_out = activated; + auto * out = t_linear(ctx, activated, out_w.tensor, out_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// Per-submodule parity taps (first text-encoder layer only). ggml tensors +// mirror torch module outputs; transpose on readback. +struct LayerTaps { + ggml_tensor * layer_in = nullptr; + ggml_tensor * attn_inproj = nullptr; // attention in_proj output + ggml_tensor * attn_qtb = nullptr; // q after permute [qh, T, H, B] + ggml_tensor * attn_ktb = nullptr; // k after permute [qh, T, H, B] + ggml_tensor * attn_scores = nullptr; // q@k before pos/bias + ggml_tensor * attn_pos = nullptr; // after pos add, before bias + ggml_tensor * attn_weights = nullptr; + ggml_tensor * ff1_inproj = nullptr; + ggml_tensor * ff1 = nullptr; + ggml_tensor * ff1_act = nullptr; + ggml_tensor * na = nullptr; + ggml_tensor * na_gated = nullptr; // after x*tanh(s) + ggml_tensor * na_gtb = nullptr; // permuted gated [T, hd, H, B] + ggml_tensor * na_attnrep = nullptr; // repeated head-0 weights + ggml_tensor * na_attended = nullptr; // after head-0 weighted matmul + ggml_tensor * na_mul = nullptr; // ch2 * y2 before out_proj + ggml_tensor * na_y2 = nullptr; // the y chunk [144, T, B] + ggml_tensor * na_h = nullptr; // in_proj output [3h, T, B] + ggml_tensor * sa1 = nullptr; + ggml_tensor * cm1_inproj = nullptr; + ggml_tensor * conv1 = nullptr; + ggml_tensor * ff2 = nullptr; + ggml_tensor * bypass_mid = nullptr; + ggml_tensor * sa2 = nullptr; + ggml_tensor * conv2 = nullptr; + ggml_tensor * ff3 = nullptr; + ggml_tensor * norm = nullptr; +}; + +// BypassModule: src_orig + (src - src_orig) * bypass_scale. +ggml_tensor * bypass(GraphBuilder & ctx, ggml_tensor * scale_t, ggml_tensor * src_orig, ggml_tensor * src) { + return modules::ScaledBypassModule().build(ctx, value(src_orig), value(src), + core::wrap_tensor(scale_t, core::TensorShape::from_dims({scale_t->ne[0]}))).tensor; +} + +// --------------------------------------------------------------------------- +// encoder layer / stack / zipformer + +ggml_tensor * encoder_layer( + GraphBuilder & ctx, + Consts & consts, + const ZipLayerWeights & w, + ggml_tensor * src, // [C, T, B] + ggml_tensor * attn_bias, // [T, 1, 1, 1] + ggml_tensor * conv_gate, // [1, T, 1] + ggml_tensor * time_row, // [C, 1, 1] or null + const ZipVoiceConfig & config, + int64_t kernel, + int64_t T, + int64_t B, + bool fm, + LayerTaps * taps = nullptr) { + const int64_t C = src->ne[0]; + const int64_t H = fm ? config.fm_num_heads : config.text_num_heads; + + auto * src_orig = src; + if (taps != nullptr) taps->layer_in = src; + ggml_tensor * attn_qtb = nullptr; + ggml_tensor * attn_ktb = nullptr; + ggml_tensor * attn_inproj = nullptr; + ggml_tensor * attn_scores = nullptr; + ggml_tensor * attn_pos = nullptr; + auto * attn_weights = attention_weights( + ctx, consts, w, src, attn_bias, H, config.query_head_dim, + config.pos_head_dim, config.pos_dim, T, B, + taps != nullptr ? &attn_qtb : nullptr, + taps != nullptr ? &attn_ktb : nullptr, + taps != nullptr ? &attn_inproj : nullptr, + taps != nullptr ? &attn_scores : nullptr, + taps != nullptr ? &attn_pos : nullptr); + if (taps != nullptr) { + taps->attn_qtb = attn_qtb; + taps->attn_ktb = attn_ktb; + taps->attn_inproj = attn_inproj; + taps->attn_scores = attn_scores; + taps->attn_pos = attn_pos; + taps->attn_weights = attn_weights; + } + + auto * x = src; + if (time_row != nullptr) { + x = t_add(ctx, x, time_row); + } + ggml_tensor * ff1_inproj = nullptr; + ggml_tensor * ff1_act = nullptr; + auto * ff1_out = feedforward(ctx, w, 1, x, T, B, &ff1_inproj, &ff1_act); + if (taps != nullptr) { taps->ff1_inproj = ff1_inproj; taps->ff1_act = ff1_act; taps->ff1 = ff1_out; } + x = t_add(ctx, x, ff1_out); + + // nonlin attention uses only head 0 + ggml_tensor * na_gated = nullptr; + ggml_tensor * na_attended = nullptr; + ggml_tensor * na_gtb = nullptr; + ggml_tensor * na_attnrep = nullptr; + ggml_tensor * na_mul = nullptr; + ggml_tensor * na_h = nullptr; + ggml_tensor * na_y2 = nullptr; + auto * na_out = nonlin_attention(ctx, w, x, attn_weights, C, H, T, B, + &na_gated, &na_attended, &na_gtb, &na_attnrep, + &na_mul, &na_h, &na_y2); + if (taps != nullptr) { + taps->na = na_out; + taps->na_gated = na_gated; + taps->na_gtb = na_gtb; + taps->na_attnrep = na_attnrep; + taps->na_attended = na_attended; + taps->na_mul = na_mul; + taps->na_y2 = na_y2; + taps->na_h = na_h; + } + x = t_add(ctx, x, na_out); + + auto * sa1_out = self_attention(ctx, w, 1, x, attn_weights, H, config.value_head_dim, T, B); + if (taps != nullptr) taps->sa1 = sa1_out; + x = t_add(ctx, x, sa1_out); + + if (time_row != nullptr) { + x = t_add(ctx, x, time_row); + } + // conv in_proj runs on the pre-conv activation (torch cm1_inproj input) + auto * cm1_out = conv_module(ctx, w, 1, x, conv_gate, C, T, B); + if (taps != nullptr) taps->conv1 = cm1_out; + x = t_add(ctx, x, cm1_out); + auto * ff2_out = feedforward(ctx, w, 2, x, T, B); + if (taps != nullptr) taps->ff2 = ff2_out; + x = t_add(ctx, x, ff2_out); + + x = bypass(ctx, w.bypass_mid_scale.tensor, src_orig, x); + if (taps != nullptr) taps->bypass_mid = x; + + auto * sa2_out = self_attention(ctx, w, 2, x, attn_weights, H, config.value_head_dim, T, B); + if (taps != nullptr) taps->sa2 = sa2_out; + x = t_add(ctx, x, sa2_out); + + if (time_row != nullptr) { + x = t_add(ctx, x, time_row); + } + auto * conv2_out = conv_module(ctx, w, 2, x, conv_gate, C, T, B); + if (taps != nullptr) taps->conv2 = conv2_out; + x = t_add(ctx, x, conv2_out); + auto * ff3_out = feedforward(ctx, w, 3, x, T, B); + if (taps != nullptr) taps->ff3 = ff3_out; + x = t_add(ctx, x, ff3_out); + + x = bias_norm(ctx, x, w.norm_bias.tensor, w.norm_log_scale); + if (taps != nullptr) taps->norm = x; + x = bypass(ctx, w.bypass_scale.tensor, src_orig, x); + return x; +} + +// SimpleDownsample: softmax-weighted sum over ds frames; pads by repeating +// the last frame. Input [C, T, B] -> [C, ceil(T/ds), B]. +ggml_tensor * downsample( + GraphBuilder & ctx, + Consts & consts, + const std::vector & softmax_bias, + ggml_tensor * x, + int64_t T, + int64_t B) { + const int64_t ds = static_cast(softmax_bias.size()); + const int64_t Td = (T + ds - 1) / ds; + const int64_t Tp = Td * ds; + const int64_t C = x->ne[0]; + + auto * x2d = t_reshape_2d(ctx, x, C, T * B); // rows (t, b) t-fastest + std::vector idx(static_cast(Tp * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t j = 0; j < Tp; ++j) { + idx[static_cast(j + b * Tp)] = + static_cast(std::min(j, T - 1) + b * T); + } + } + auto * gather = consts.i32(idx, {Tp * B}); + auto * padded = t_rows(ctx, x2d, gather); // [C, Tp*B] + + // weighted sum over the ds sub-frame axis + auto * windowed = t_reshape_4d(ctx, padded, C, ds, Td, B); + auto * w_frame = t_permute(ctx, windowed, 3, 0, 1, 2); // [ds, Td, B, C] + auto * w_frame_c = t_cont(ctx, w_frame); + auto * w2 = t_reshape_3d(ctx, w_frame_c, ds, Td * B, C); + auto * weights2 = consts.f32(softmax_bias, {ds, 1}); + auto * summed = t_matmul(ctx, weights2, w2); // [1, Td*B, C] + auto * as_c = t_permute(ctx, summed, 1, 2, 0, 3); // [C, 1, Td*B, 1] + auto * as_c2 = t_cont(ctx, as_c); + auto * out = t_reshape_3d(ctx, as_c2, C, Td, B); + return out; +} + +// SimpleUpsample: repeat each frame ds times, then truncate to T. +ggml_tensor * upsample( + GraphBuilder & ctx, + Consts & consts, + int64_t ds, + int64_t T, + ggml_tensor * x, // [C, Td, B] + int64_t B) { + const int64_t Td = x->ne[1]; + const int64_t C = x->ne[0]; + auto * x2d = t_reshape_2d(ctx, x, C, Td * B); + const int64_t Tp = Td * ds; + std::vector idx(static_cast(Tp * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t i = 0; i < Tp; ++i) { + idx[static_cast(i + b * Tp)] = + static_cast(i / ds + b * Td); + } + } + auto * gather = consts.i32(idx, {Tp * B}); + auto * out = t_rows(ctx, x2d, gather); // [C, Tp*B] contiguous + if (Tp == T) { + return t_reshape_3d(ctx, out, C, T, B); + } + // truncate to T frames: rows are (frame, batch) frame-fastest + return ggml_view_3d(ctx, out, C, T, B, + C * sizeof(float), static_cast(Tp) * C * sizeof(float), 0); +} + +ggml_tensor * zipformer_forward( + GraphBuilder & ctx, + Consts & consts, + const TTSZipformerWeights & w, + const ZipVoiceConfig & config, + ggml_tensor * input, // [in_dim, T, B] + ggml_tensor * time_emb, // [time_embed_dim] or null + ggml_tensor * const * pad_bias, + ggml_tensor * const * conv_gate, + bool fm, + int64_t B, + std::vector * layer_taps = nullptr, + LayerTaps * first_layer_taps = nullptr) { + const bool with_time = fm; + auto * x = t_linear(ctx, input, w.in_proj_w.tensor, w.in_proj_b.tensor); // [C, T, B] + + ggml_tensor * outer_time = nullptr; + if (with_time && time_emb != nullptr) { + auto * h = t_linear(ctx, time_emb, w.time_mlp0_w.tensor, w.time_mlp0_b.tensor); + outer_time = t_linear(ctx, swoosh_r(ctx, h), w.time_mlp2_w.tensor, w.time_mlp2_b.tensor); + } + + const int64_t T = x->ne[1]; + for (size_t s = 0; s < w.stacks.size(); ++s) { + auto & stack = w.stacks[s]; + const int64_t ds = fm ? config.fm_downsampling_factor[s] : 1; + const int64_t kernel = fm ? config.fm_cnn_kernel[s] : config.text_cnn_kernel; + + ggml_tensor * time_row = nullptr; + if (with_time && stack.time_proj_w.tensor != nullptr) { + // torch: encoders[i].time_emb = Sequential(SwooshR, Linear) + auto * activated = swoosh_r(ctx, outer_time); + auto * projected = t_linear(ctx, activated, stack.time_proj_w.tensor, stack.time_proj_b.tensor); + time_row = t_reshape_3d(ctx, projected, projected->ne[0], 1, 1); + } + + ggml_tensor * stack_in = x; + if (ds > 1) { + stack_in = downsample(ctx, consts, stack.downsample_bias, x, T, B); + } + const int64_t Ts = stack_in->ne[1]; + + auto * h = stack_in; + for (size_t li = 0; li < stack.layers.size(); ++li) { + h = encoder_layer( + ctx, consts, stack.layers[li], h, pad_bias[s], conv_gate[s], time_row, + config, kernel, Ts, B, fm, + first_layer_taps != nullptr && s == 0 && li == 0 ? first_layer_taps : nullptr); + if (layer_taps != nullptr) { + layer_taps->push_back(h); + } + } + + if (ds > 1) { + auto * up = upsample(ctx, consts, ds, T, h, B); + x = bypass(ctx, stack.out_combiner_scale.tensor, x, up); + } else { + x = h; + } + } + auto * out = t_linear(ctx, x, w.out_proj_w.tensor, w.out_proj_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], out->ne[1], B, 1); +} + +struct GraphBuildResult { + ggml_context * ctx = nullptr; + ggml_cgraph * graph = nullptr; +}; + +GraphBuildResult finish_graph( + ggml_context * ctx, + ggml_context * tensor_ctx, + ggml_tensor * output, + const std::vector & extra_roots, + std::vector & staged, + size_t node_budget, + ggml_backend_t backend, + void ** gallocr_out, + ggml_backend_buffer_t * buffer_out) { + GraphBuildResult result; + result.ctx = ctx; + result.graph = ggml_new_graph_custom(ctx, node_budget, false); + ggml_build_forward_expand(result.graph, output); + // ALL extra outputs must be expanded BEFORE the arena reservation: + // nodes appended after ggml_gallocr_alloc_graph never get memory. + for (ggml_tensor * tap : extra_roots) { + if (tap != nullptr) { + ggml_build_forward_expand(result.graph, tap); + } + } + if (backend == nullptr) { + throw std::runtime_error("zipvoice: graph backend is required"); + } + // Half-staged matrix products accumulate audible error across the flow + // solver steps. Keep the CPU oracle unchanged and request F32 on GPUs. + if (core::backend_type(backend) != core::BackendType::Cpu) { + for (int i = 0; i < ggml_graph_n_nodes(result.graph); ++i) { + auto * node = ggml_graph_node(result.graph, i); + if (node->op == GGML_OP_MUL_MAT) ggml_mul_mat_set_prec(node, GGML_PREC_F32); + } + } + core::validate_backend_graph_supported(backend, result.graph, "zipvoice"); + // private storage for leaves + constants FIRST: with a buffer already + // assigned, gallocr skips these tensors instead of arena-aliasing them + ggml_backend_buffer_t buffer = + ggml_backend_alloc_ctx_tensors(tensor_ctx, backend); + if (buffer == nullptr) { + throw std::runtime_error("zipvoice: tensor buffer allocation failed"); + } + *buffer_out = buffer; + auto * gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + *gallocr_out = gallocr; + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, result.graph) || + !ggml_gallocr_alloc_graph(gallocr, result.graph)) { + throw std::runtime_error("zipvoice: graph allocation failed"); + } + for (auto & item : staged) { + ggml_backend_tensor_set(item.tensor, item.bytes.data(), 0, item.bytes.size()); + } + staged.clear(); + return result; +} + +} // namespace + +FmDecoderGraph build_fm_decoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t T, + int64_t B, + bool with_guidance, + bool cuda_backend, + ggml_backend_t backend) { + (void)cuda_backend; + const size_t ctx_bytes = 512ULL * 1024ULL * 1024ULL; + ggml_context * ctx = ggml_init({ctx_bytes, nullptr, true}); + // leaves + constants get their own context and backend buffer so the + // graph arena (gallocr) never aliases them (see skill: gallocr aliasing) + ggml_context * tensor_ctx = ggml_init({64ULL * 1024ULL * 1024ULL, nullptr, true}); + FmDecoderGraph g; + g.ctx = ctx; + g.tensor_ctx = tensor_ctx; + g.T = T; + g.B = B; + g.cuda = false; + g.backend = backend; + + std::vector staged; + Consts consts{tensor_ctx, &staged}; + + auto * x_cat = ggml_new_tensor_3d(tensor_ctx, GGML_TYPE_F32, config.feat_dim * 3, T, B); + ggml_set_input(x_cat); + g.x_cat = x_cat; + + auto * time_emb = ggml_new_tensor_1d(tensor_ctx, GGML_TYPE_F32, config.time_embed_dim); + ggml_set_input(time_emb); + g.time_emb = time_emb; + + if (with_guidance) { + auto * guidance = ggml_new_tensor_1d(tensor_ctx, GGML_TYPE_F32, config.time_embed_dim); + ggml_set_input(guidance); + g.guidance_emb = guidance; + } + + // per-stack masks. Downsampling factors are relative to the full frame + // rate (U-Net style), so stack s runs at ceil(T / ds[s]) frames. + for (size_t s = 0; s < weights.fm_decoder.stacks.size(); ++s) { + const int64_t ds = config.fm_downsampling_factor[s]; + const int64_t T_s = (T + ds - 1) / ds; + auto * bias = ggml_new_tensor_4d(tensor_ctx, GGML_TYPE_F32, T_s, 1, 1, 1); + ggml_set_input(bias); + g.pad_bias[s] = bias; + auto * gate = ggml_new_tensor_3d(tensor_ctx, GGML_TYPE_F32, 1, T_s, 1); + ggml_set_input(gate); + g.conv_gate[s] = gate; + } + + GraphBuilder builder(ctx, core::backend_type(backend)); + ggml_tensor * time_input = g.time_emb; + if (with_guidance && weights.fm_decoder.guidance_embed_w.tensor != nullptr) { + auto * guidance_proj = t_linear( + builder, g.guidance_emb, weights.fm_decoder.guidance_embed_w.tensor, nullptr); + time_input = t_add(builder, g.time_emb, guidance_proj); + } + + auto * out = zipformer_forward( + builder, consts, weights.fm_decoder, config, x_cat, time_input, + g.pad_bias, g.conv_gate, true, B); + ggml_set_output(out); + g.output = out; + + auto built = finish_graph(ctx, tensor_ctx, out, {}, staged, 262144, backend, &g.gallocr, &g.buffer); + g.graph = built.graph; + return g; +} + +TextEncoderGraph build_text_encoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t S, + bool cuda_backend, + ggml_backend_t backend) { + (void)cuda_backend; + const size_t ctx_bytes = 256ULL * 1024ULL * 1024ULL; + ggml_context * ctx = ggml_init({ctx_bytes, nullptr, true}); + ggml_context * tensor_ctx = ggml_init({16ULL * 1024ULL * 1024ULL, nullptr, true}); + TextEncoderGraph g; + g.ctx = ctx; + g.tensor_ctx = tensor_ctx; + g.S = S; + g.cuda = false; + g.backend = backend; + + std::vector staged; + Consts consts{tensor_ctx, &staged}; + + auto * ids = ggml_new_tensor_1d(tensor_ctx, GGML_TYPE_I32, S); + ggml_set_input(ids); + g.token_ids = ids; + + auto * bias = ggml_new_tensor_4d(tensor_ctx, GGML_TYPE_F32, S, 1, 1, 1); + ggml_set_input(bias); + g.pad_bias[0] = bias; + auto * gate = ggml_new_tensor_3d(tensor_ctx, GGML_TYPE_F32, 1, S, 1); + ggml_set_input(gate); + g.conv_gate[0] = gate; + + GraphBuilder builder(ctx, core::backend_type(backend)); + auto * embedded = t_rows(builder, weights.text_encoder.embed_w.tensor, ids); + std::vector layer_taps; + LayerTaps first_layer; + auto * out = zipformer_forward( + builder, consts, weights.text_encoder, config, embedded, nullptr, + g.pad_bias, g.conv_gate, false, 1, &layer_taps, &first_layer); + ggml_set_output(out); + g.output = out; + std::vector extra_roots; + const auto mark_output = [](ggml_tensor * tap) { + ggml_set_output(tap); + // reshape results are VIEWS: the gallocr frees/reuses the underlying + // node unless it is flagged too (graph outputs are never freed). + if (tap->view_src != nullptr) { + ggml_set_output(tap->view_src); + } + }; + for (auto * tap : layer_taps) { + mark_output(tap); + g.layer_taps.push_back(tap); + extra_roots.push_back(tap); + } + for (ggml_tensor * tap : { + first_layer.layer_in, first_layer.attn_qtb, first_layer.attn_ktb, + first_layer.attn_inproj, first_layer.attn_scores, + first_layer.attn_pos, first_layer.attn_weights, first_layer.ff1_inproj, + first_layer.ff1_act, first_layer.ff1, first_layer.na, first_layer.na_gated, + first_layer.na_gtb, first_layer.na_attnrep, first_layer.na_attended, first_layer.na_mul, first_layer.na_y2, first_layer.na_h, first_layer.sa1, first_layer.cm1_inproj, + first_layer.conv1, first_layer.ff2, first_layer.bypass_mid, + first_layer.sa2, first_layer.conv2, first_layer.ff3, first_layer.norm}) { + if (tap != nullptr) { + mark_output(tap); + extra_roots.push_back(tap); + } + } + + auto built = finish_graph(ctx, tensor_ctx, out, extra_roots, staged, 262144, backend, &g.gallocr, &g.buffer); + g.stage_taps.clear(); + for (ggml_tensor * tap : { + first_layer.layer_in, first_layer.attn_qtb, first_layer.attn_ktb, + first_layer.attn_inproj, first_layer.attn_scores, + first_layer.attn_pos, first_layer.attn_weights, first_layer.ff1_inproj, + first_layer.ff1_act, first_layer.ff1, first_layer.na, first_layer.na_gated, + first_layer.na_gtb, first_layer.na_attnrep, first_layer.na_attended, first_layer.na_mul, first_layer.na_y2, first_layer.na_h, first_layer.sa1, first_layer.cm1_inproj, + first_layer.conv1, first_layer.ff2, first_layer.bypass_mid, + first_layer.sa2, first_layer.conv2, first_layer.ff3, first_layer.norm}) { + if (tap != nullptr) g.stage_taps.push_back(tap); + } + g.graph = built.graph; + return g; +} + +} // namespace engine::models::zipvoice diff --git a/src/framework/audio/espeak_phonemizer.cpp b/src/framework/audio/espeak_phonemizer.cpp index 58c2b995b..e8f895144 100644 --- a/src/framework/audio/espeak_phonemizer.cpp +++ b/src/framework/audio/espeak_phonemizer.cpp @@ -79,8 +79,13 @@ struct Runtime { #ifdef _WIN32 "espeak-ng.dll", "libespeak-ng.dll", #elif defined(__APPLE__) + "/opt/homebrew/lib/libespeak-ng.dylib", + "/usr/local/lib/libespeak-ng.dylib", "libespeak-ng.dylib", "libespeak-ng.1.dylib", #else + "/usr/lib/x86_64-linux-gnu/libespeak-ng.so.1", + "/usr/lib/aarch64-linux-gnu/libespeak-ng.so.1", + "/usr/lib/libespeak-ng.so.1", "libespeak-ng.so.1", "libespeak-ng.so", #endif }); @@ -130,6 +135,27 @@ EspeakPhonemizer::EspeakPhonemizer(std::filesystem::path library, #endif if (data_.extension() == ".bin" || data_.extension() == ".gguf") data_ = materialize_espeak_data(data_); +#if !defined(AUDIOCPP_STATIC_ESPEAK) && !defined(_WIN32) + // Dynamic eSpeak-ng with no explicit data path: probe common install + // locations (library-relative first, then system share directories) so a + // stock install works without session options. + if (data_.empty()) { + std::vector data_candidates; + if (!library_.empty()) { + const auto lib_dir = library_.parent_path(); + data_candidates.push_back(lib_dir / ".." / "share" / "espeak-ng-data"); + data_candidates.push_back(lib_dir / "espeak-ng-data"); + } + data_candidates.push_back("/opt/homebrew/share/espeak-ng-data"); + data_candidates.push_back("/usr/local/share/espeak-ng-data"); + data_candidates.push_back("/usr/share/espeak-ng-data"); + for (const auto & candidate : data_candidates) + if (std::filesystem::is_regular_file(candidate / "phontab")) { + data_ = candidate.lexically_normal(); + break; + } + } +#endif if (!data_.empty() && data_.filename().empty()) data_ = data_.parent_path(); if (!library_.empty() && !std::filesystem::is_regular_file(library_)) throw std::runtime_error("eSpeak-ng library does not exist: " + library_.string()); diff --git a/src/framework/modules/convnext_modules.cpp b/src/framework/modules/convnext_modules.cpp new file mode 100644 index 000000000..b4d29d607 --- /dev/null +++ b/src/framework/modules/convnext_modules.cpp @@ -0,0 +1,23 @@ +#include "engine/framework/modules/convnext_modules.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" + +namespace engine::modules { +core::TensorValue build_convnext1d(core::ModuleBuildContext & ctx, + const core::TensorValue & input, const ConvNeXt1dWeights & weights) { + core::validate_rank_between(input, 3, 3, "ConvNeXt1d input"); + const int64_t channels = input.shape.last_dim(); + const int64_t kernel = weights.depthwise.weight.shape.last_dim(); + const int64_t hidden = weights.expansion.weight.shape.at(0); + auto x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, input); + x = DepthwiseConv1dModule({channels, kernel, 1, int(kernel / 2), 1, true}).build(ctx, x, weights.depthwise); + x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, x); + x = LayerNormModule({channels, 1e-6f, true, true}).build(ctx, x, weights.norm); + x = LinearModule({channels, hidden, true}).build(ctx, x, weights.expansion); + x = GeluModule({GeluApproximation::ExactErf}).build(ctx, x); + x = LinearModule({hidden, channels, true}).build(ctx, x, weights.projection); + x = MulModule().build(ctx, x, RepeatModule({x.shape}).build(ctx, core::reshape_tensor(ctx, weights.gamma, core::TensorShape::from_dims({1, 1, channels})))); + return AddModule().build(ctx, input, x); +} +} diff --git a/src/framework/modules/vocoders/vocos_vocoder.cpp b/src/framework/modules/vocoders/vocos_vocoder.cpp new file mode 100644 index 000000000..1182325cb --- /dev/null +++ b/src/framework/modules/vocoders/vocos_vocoder.cpp @@ -0,0 +1,147 @@ +#include "engine/framework/modules/vocoders/vocos_vocoder.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/audio/istft_graph.h" +#include "engine/framework/modules/convnext_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/runtime/cache_slots.h" +#include "ggml-alloc.h" +#include +#include +#include + +namespace engine::modules { +namespace { +constexpr int kMel = 100, kFFT = 1024, kHop = 256; +struct Graph { + ggml_backend_t backend = nullptr; + ggml_context * ctx = nullptr; + ggml_context * inputs = nullptr; + ggml_backend_buffer_t buffer = nullptr; + ggml_gallocr_t arena = nullptr; + ggml_cgraph * graph = nullptr; + ggml_tensor * mel = nullptr; + ggml_tensor * spec = nullptr; + std::unique_ptr istft; + ~Graph() { + if (backend && graph) core::release_backend_graph_resources(backend, graph, true); + if (arena) ggml_gallocr_free(arena); + if (buffer) ggml_backend_buffer_free(buffer); + if (ctx) ggml_free(ctx); + if (inputs) ggml_free(inputs); + } +}; +} +core::TensorValue build_vocos_backbone(core::ModuleBuildContext & ctx, + const core::TensorValue & mel, const VocosBackboneWeights & weights) { + const auto channels = weights.embed.weight.shape.at(0); + const auto kernel = weights.embed.weight.shape.last_dim(); + auto x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, mel); + x = Conv1dModule({mel.shape.last_dim(), channels, kernel, 1, int(kernel / 2), 1, true}).build(ctx, x, weights.embed); + x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, x); + x = LayerNormModule({channels, 1e-6f, true, true}).build(ctx, x, weights.input_norm); + for (const auto & block : weights.blocks) x = build_convnext1d(ctx, x, block); + x = LayerNormModule({channels, 1e-6f, true, true}).build(ctx, x, weights.final_norm); + return LinearModule({channels, weights.head.weight.shape.at(0), true}).build(ctx, x, weights.head); +} +class VocosVocoder::Impl { +public: + ggml_backend_t backend; + int threads; + core::BackendType type; + core::BackendWeightStore store; + VocosBackboneWeights weights; + std::vector window; + runtime::CacheSlots> graphs{1}; + + Impl(const std::string & checkpoint, ggml_backend_t b, int n) + : backend(b), threads(n > 0 ? n : 1), type(core::backend_type(b)), + store(b, type, "vocos", 8ULL << 20) { + auto source = assets::open_tensor_source(checkpoint); + if (std::filesystem::path(checkpoint).extension() == ".gguf") { + source = assets::make_prefixed_tensor_source(source, "vocos"); + } + const auto f32 = [&](const std::string & name) { + return store.load_f32_tensor(*source, name, source->require_metadata(name).shape); + }; + const auto norm = [&](const std::string & name) -> NormWeights { + return {f32(name + ".weight"), f32(name + ".bias")}; + }; + const auto linear = [&](const std::string & name) -> LinearWeights { + // Keep matrix weights quantized; convolution and norm weights stay F32. + const auto weight = name + ".weight"; + return {store.load_tensor(*source, weight, assets::TensorStorageType::Native, + source->require_metadata(weight).shape), + f32(name + ".bias")}; + }; + weights.embed = {f32("backbone.embed.weight"), f32("backbone.embed.bias")}; + weights.input_norm = norm("backbone.norm"); + weights.final_norm = norm("backbone.final_layer_norm"); + weights.head = linear("head.out"); + for (int i = 0; source->has_tensor("backbone.convnext." + std::to_string(i) + ".dwconv.weight"); ++i) { + const auto p = "backbone.convnext." + std::to_string(i); + weights.blocks.push_back({{f32(p + ".dwconv.weight"), f32(p + ".dwconv.bias")}, + norm(p + ".norm"), linear(p + ".pwconv1"), linear(p + ".pwconv2"), f32(p + ".gamma")}); + } + if (weights.blocks.empty()) throw std::runtime_error("Vocos has no ConvNeXt blocks"); + store.upload(); + source->release_storage(); + window.resize(kFFT); + for (int i = 0; i < kFFT; ++i) window[i] = 0.5f * (1 - std::cos(2.0f * 3.14159265358979323846f * i / kFFT)); + } + + std::unique_ptr build(int64_t frames) { + auto g = std::make_unique(); + g->backend = backend; + g->ctx = ggml_init({16ULL << 20, nullptr, true}); + g->inputs = ggml_init({1ULL << 20, nullptr, true}); + if (!g->ctx || !g->inputs) throw std::runtime_error("Vocos context allocation failed"); + core::ModuleBuildContext ctx{g->ctx, "vocos", type}, io{g->inputs, "vocos.input", type}; + auto mel = core::make_tensor(io, GGML_TYPE_F32, core::TensorShape::from_dims({1, frames, kMel})); + g->mel = mel.tensor; + ggml_set_input(g->mel); + auto x = build_vocos_backbone(ctx, mel, weights); + g->spec = x.tensor; + ggml_set_output(g->spec); + g->graph = ggml_new_graph_custom(g->ctx, 8192, false); + ggml_build_forward_expand(g->graph, g->spec); + if (type != core::BackendType::Cpu) { + for (int i = 0; i < ggml_graph_n_nodes(g->graph); ++i) { + auto * node = ggml_graph_node(g->graph, i); + if (node->op == GGML_OP_MUL_MAT) ggml_mul_mat_set_prec(node, GGML_PREC_F32); + } + } + g->buffer = ggml_backend_alloc_ctx_tensors(g->inputs, backend); + g->arena = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (!g->buffer || !g->arena || !ggml_gallocr_alloc_graph(g->arena, g->graph)) + throw std::runtime_error("Vocos graph allocation failed"); + g->istft = std::make_unique( + audio::HostLogMagnitudePhaseISTFTConfig{frames, kFFT, kHop, kFFT + 2, size_t(threads)}); + return g; + } + std::vector decode(const std::vector & mel) { + if (mel.size() < 2 * kMel || mel.size() % kMel) throw std::invalid_argument("Vocos needs complete 100-bin mel frames"); + const int64_t frames = mel.size() / kMel; + if (!graphs.find(frames)) { + // Release the previous arena before allocating the next shape. + graphs.clear(); + graphs.put(frames, build(frames)); + } + auto & g = **graphs.find(frames); + ggml_backend_tensor_set(g.mel, mel.data(), 0, mel.size() * sizeof(float)); + core::set_backend_threads(backend, threads); + if (core::compute_backend_graph(backend, g.graph) != GGML_STATUS_SUCCESS) throw std::runtime_error("Vocos compute failed"); + ggml_backend_synchronize(backend); + auto spec = core::read_tensor_f32(g.spec); + auto audio = g.istft->compute(spec, window).audio; + // Framework ISTFT uses same padding (T*hop). Vocos uses center=True: + // trim a further half hop at each end to obtain (T-1)*hop samples. + return {audio.begin() + kHop / 2, audio.end() - kHop / 2}; + } +}; +VocosVocoder::VocosVocoder(const std::string & path, ggml_backend_t backend, int threads) + : impl_(std::make_unique(path, backend, threads)) {} +VocosVocoder::~VocosVocoder() = default; +std::vector VocosVocoder::decode(const std::vector & mel) { return impl_->decode(mel); } +size_t VocosVocoder::cached_graph_count() const { return impl_->graphs.size(); } +} diff --git a/tests/zipvoice/build_reference.py b/tests/zipvoice/build_reference.py new file mode 100644 index 000000000..9eba604aa --- /dev/null +++ b/tests/zipvoice/build_reference.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Generate parity references from an upstream ZipVoice checkout and fixed inputs. + +No downloads or text frontend are needed. Run with the upstream Python environment: + python build_reference.py --repo /path/to/ZipVoice --model-dir /path/to/zipvoice_distill \ + --fixture moonlight_distill_case.npz --output reference.npz [--vocos model.safetensors] + +Hooks capture actual masked module outputs, never a hand-recomputed approximation. +The supplied fixture's text_condition/x1 are retained as independent golden outputs. +""" +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +import torch + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--fixture", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--vocos", type=Path) + args = parser.parse_args() + sys.path.insert(0, str(args.repo.resolve())) + from zipvoice.models.zipvoice_distill import ZipVoiceDistill + from zipvoice.utils.feature import VocosFbank + + torch.set_num_threads(4) + config = json.loads((args.model_dir / "model.json").read_text()) + vocab = dict(line.rstrip("\n").split("\t") for line in + (args.model_dir / "tokens.txt").read_text().splitlines()) + model = ZipVoiceDistill(**config["model"], vocab_size=len(vocab), pad_id=int(vocab["_"])) + state = torch.load(args.model_dir / "model.pt", map_location="cpu", weights_only=False)["model"] + model.load_state_dict(state, strict=True) + model.eval() + with np.load(args.fixture) as fixture: + outputs = {key: fixture[key] for key in fixture.files} + + def capture(name): + def hook(module, inputs, output): + outputs[name] = output.detach().clone().cpu().numpy().astype(np.float32) + return hook + + hooks = [] + for i, layer in enumerate(model.text_encoder.encoders[0].layers): + hooks.append(layer.register_forward_hook(capture(f"layer{i}"))) + first = model.text_encoder.encoders[0].layers[0] + for name in ("self_attn_weights", "feed_forward1", "nonlin_attention", "self_attn1", + "conv_module1", "feed_forward2", "bypass_mid", "self_attn2", + "conv_module2", "feed_forward3", "norm"): + hooks.append(dict(first.named_modules())[name].register_forward_hook(capture(name))) + for name, key in (("nonlin_attention.in_proj", "na_h_ref"), + ("nonlin_attention.identity1", "na_gated_ref"), + ("nonlin_attention.identity2", "na_y2_ref"), + ("nonlin_attention.identity3", "na_mul_ref")): + hooks.append(dict(first.named_modules())[name].register_forward_hook(capture(key))) + with torch.inference_mode(): + tokens = outputs["tokens"].tolist() + prompt_tokens = outputs["prompt_tokens"].tolist() + embed, _ = model.forward_text_embed([prompt_tokens[0] + tokens[0]]) + outputs["embed"] = embed.numpy() + for hook in hooks: + hook.remove() + tc, mask = model.forward_text_inference_ratio_duration( + tokens=tokens, prompt_tokens=prompt_tokens, + prompt_features_lens=torch.from_numpy(outputs["prompt_features_lens"]), speed=1.0) + np.testing.assert_allclose(tc.numpy(), outputs["text_condition"], atol=2e-6, rtol=2e-5) + # Odd and even lengths exercise repeat-last padding and downsampling. + for length in (17, 32, outputs["x0"].shape[1]): + for step, time in enumerate((0.0, 0.6)): + key = f"velocity_{length}_{step}" + result = model.forward_fm_decoder( + t=torch.tensor(time), xt=torch.from_numpy(outputs["x0"][:, :length]), + text_condition=tc[:, :length], + speech_condition=torch.from_numpy(outputs["speech_condition"][:, :length]), + padding_mask=mask[:, :length], guidance_scale=torch.tensor(3.0)) + outputs[key] = result.numpy() + # Different conditions in each batch catch accidental head-0/batch-0 reuse. + length = 17 + batch_x = torch.from_numpy(outputs["x0"][:, :length]).repeat(2, 1, 1) + batch_text = torch.cat([torch.zeros_like(tc[:, :length]), tc[:, :length]]) + batch_speech = torch.from_numpy(outputs["speech_condition"][:, :length]).repeat(2, 1, 1) + outputs["batch_x"] = batch_x.numpy() + outputs["batch_text"] = batch_text.numpy() + outputs["batch_speech"] = batch_speech.numpy() + outputs["batch_velocity"] = model.forward_fm_decoder( + t=torch.tensor(0.6), xt=batch_x, text_condition=batch_text, + speech_condition=batch_speech, padding_mask=mask[:, :length].repeat(2, 1), + guidance_scale=torch.tensor(3.0)).numpy() + # Broadband input keeps quiet high-frequency bins numerically well conditioned. + wav = torch.from_numpy(np.random.default_rng(123).normal(0, 0.1, 4096).astype(np.float32)) + outputs["fbank_wav"] = wav.numpy() + outputs["fbank_mel"] = VocosFbank().extract(wav, sampling_rate=24000).numpy() + if args.vocos: + from safetensors.torch import load_file + from vocos.models import VocosBackbone + from vocos.heads import ISTFTHead + backbone = VocosBackbone(input_channels=100, dim=512, intermediate_dim=1536, num_layers=8).eval() + head = ISTFTHead(dim=512, n_fft=1024, hop_length=256, padding="center").eval() + weights = (load_file(str(args.vocos)) if args.vocos.suffix == ".safetensors" + else torch.load(args.vocos, map_location="cpu", weights_only=True)) + backbone.load_state_dict({k.removeprefix("backbone."): v for k, v in weights.items() if k.startswith("backbone.")}) + head.load_state_dict({k.removeprefix("head."): v for k, v in weights.items() if k.startswith("head.")}) + # Skip the leading silence in the reference recording. + mel = torch.from_numpy(outputs["prompt_features"][:, 32:64]) / 0.1 + outputs["vocos_mel"] = mel.numpy() + outputs["vocos_audio"] = head(backbone(mel.transpose(1, 2))).numpy() + assert np.sqrt(np.mean(outputs["vocos_audio"] ** 2)) > 1e-3, "Vocos fixture must contain speech" + args.output.parent.mkdir(parents=True, exist_ok=True) + np.savez(args.output, **outputs) + print(f"Wrote {args.output}: {len(outputs)} arrays") + + +if __name__ == "__main__": + main() diff --git a/tests/zipvoice/zh_reference_ids.py b/tests/zipvoice/zh_reference_ids.py new file mode 100644 index 000000000..3824eecde --- /dev/null +++ b/tests/zipvoice/zh_reference_ids.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Dump reference EmiliaTokenizer token ids for the zh-token parity test. + +Run with the upstream ZipVoice Python environment: + /path/to/ZipVoice/.venv/bin/python tests/zipvoice/zh_reference_ids.py \ + --tokenizer-repo /path/to/ZipVoice \ + --model-dir /models/ZipVoice/zipvoice_distill \ + --output /tmp/zipvoice/zh_reference_ids.json +""" +import argparse +import json +import sys +from pathlib import Path + + +CORPUS = [ + "今夜的月光如此清亮，不做些什么真是浪费。随我一同去月下漫步吧，不许拒绝。", + "你好，世界。", + "老虎养殖场里养着几只老虎。", + "一点诚意都没有，不见不散。", + "一直走，不要停，第一个路口右转。", + "我在银行门口的长凳上休息了很久。", + "重庆的重量单位和别处不一样吗？", + "2024年3月14日，第2名，50元。", + "我用 audio.cpp 合成语音，效果不错！", + "This is English text inside.", + " 人最棒。", + "他拿着画，走向画室。", + "买东西的时候记得带伞。", + "地得工工作很努力。", +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokenizer-repo", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + import logging + logging.disable(logging.CRITICAL) + sys.path.insert(0, str(args.tokenizer_repo.resolve())) + from zipvoice.tokenizer.tokenizer import EmiliaTokenizer + + tokenizer = EmiliaTokenizer(token_file=str(args.model_dir / "tokens.txt")) + out = {} + for text in CORPUS: + ids = tokenizer.texts_to_token_ids([text])[0] + tokens = tokenizer.texts_to_tokens([text])[0] + out[text] = {"ids": ids, "tokens": tokens} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8") + print(f"wrote {args.output} ({len(out)} cases)") + + +if __name__ == "__main__": + main() diff --git a/tests/zipvoice/zipvoice_parity_main.cpp b/tests/zipvoice/zipvoice_parity_main.cpp new file mode 100644 index 000000000..aeaddacc3 --- /dev/null +++ b/tests/zipvoice/zipvoice_parity_main.cpp @@ -0,0 +1,408 @@ +// ZipVoice parity harness against the moonlight fixture produced by the +// zipvoice-lite MLX port (zipvoice/mlx/fixtures/moonlight_distill_case.npz). +// The fixture stores the PyTorch reference inputs and outputs for the +// ZipVoice-Distill checkpoint: +// tokens, prompt_tokens, prompt_features(+lens), x0, x1 (full sample), +// text_condition (text encoder + duration prediction output). +// Usage: zipvoice_parity +#include "engine/community_models/zipvoice/synthesize.h" +#include "engine/framework/audio/wav_writer.h" +#include "engine/framework/model_spec/package.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// ---- minimal ZIP (STORED) + .npy reader ----------------------------------- + +struct NpyArray { + std::string name; + std::vector shape; + char typechar = 'f'; // 'f' float32, 'i' int32 + std::vector data; +}; + +uint16_t rd16(const uint8_t * p) { return uint16_t(p[0]) | (uint16_t(p[1]) << 8); } +uint32_t rd32(const uint8_t * p) { + return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24); +} + +bool parse_npy(const uint8_t * raw, size_t size, NpyArray & arr) { + if (size < 10 || std::memcmp(raw, "\x93NUMPY", 6) != 0) { + std::fprintf(stderr, "bad npy magic\n"); + std::exit(2); + } + const uint16_t header_len = rd16(raw + 8); + const std::string header(reinterpret_cast(raw + 10), header_len); + const size_t data_offset = 10 + header_len; + // 'descr': '(header[p]))) { + int64_t v = 0; + while (std::isdigit(static_cast(header[p]))) { + v = v * 10 + (header[p] - '0'); + ++p; + } + arr.shape.push_back(v); + } else { + ++p; + } + } + arr.data.assign(raw + data_offset, raw + size); + return true; +} + +std::vector load_npz(const std::string & path) { + std::ifstream f(path, std::ios::binary); + if (!f) { std::fprintf(stderr, "cannot open %s\n", path.c_str()); std::exit(1); } + std::vector zip((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + // find EOCD + size_t eocd = zip.size() - 22; + while (eocd > 0 && rd32(&zip[eocd]) != 0x06054b50) --eocd; + if (rd32(&zip[eocd]) != 0x06054b50) { std::fprintf(stderr, "bad npz\n"); std::exit(2); } + const uint16_t entries = rd16(&zip[eocd + 10]); + const uint32_t cd_offset = rd32(&zip[eocd + 16]); + // collect (name, local offset) in directory order + struct Entry { std::string name; uint32_t local; }; + std::vector directory; + uint32_t off = cd_offset; + for (uint16_t i = 0; i < entries; ++i) { + if (rd32(&zip[off]) != 0x02014b50) break; + const uint16_t name_len = rd16(&zip[off + 28]); + const uint16_t extra_len = rd16(&zip[off + 30]); + const uint16_t comment_len = rd16(&zip[off + 32]); + const uint32_t local = rd32(&zip[off + 42]); + directory.push_back({std::string(reinterpret_cast(&zip[off + 46]), name_len), local}); + off += 46 + name_len + extra_len + comment_len; + } + std::vector arrays; + for (size_t i = 0; i < directory.size(); ++i) { + const uint32_t local = directory[i].local; + const std::string & name = directory[i].name; + uint32_t csize = rd32(&zip[local + 18]); + const size_t data_start = local + 30 + rd16(&zip[local + 26]) + rd16(&zip[local + 28]); + if (csize == 0xFFFFFFFFu) { + // ZIP64 sentinel: derive the size from the neighbouring entry + // (STORED entries are laid out contiguously). + const size_t next = (i + 1 < directory.size()) ? directory[i + 1].local : cd_offset; + if (next < data_start) { std::fprintf(stderr, "bad npz entry %s\n", name.c_str()); std::exit(2); } + csize = static_cast(next - data_start); + } + if (name.size() > 4 && name.substr(name.size() - 4) == ".npy") { + NpyArray arr; + if (parse_npy(&zip[data_start], csize, arr)) { + arr.name = name.substr(0, name.size() - 4); + arrays.push_back(std::move(arr)); + } + } + } + return arrays; +} + +double cosine(const std::vector & a, const std::vector & b) { + double dot = 0, na = 0, nb = 0; + const size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) { + dot += double(a[i]) * double(b[i]); + na += double(a[i]) * double(a[i]); + nb += double(b[i]) * double(b[i]); + } + return dot / (std::sqrt(na) * std::sqrt(nb) + 1e-12); +} + +double max_abs_diff(const std::vector & a, const std::vector & b) { + double m = 0; + const size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) m = std::max(m, double(std::fabs(a[i] - b[i]))); + return m; +} + +bool check(const std::string & name, const std::vector & got, + const std::vector & want, float atol, float rtol) { + bool ok = !got.empty() && got.size() == want.size(); + double squared = 0; + for (size_t i = 0; i < std::min(got.size(), want.size()); ++i) { + const double diff = std::abs(double(got[i]) - want[i]); + squared += diff * diff; + ok = ok && std::isfinite(got[i]) && std::isfinite(want[i]) && + diff <= atol + rtol * std::abs(want[i]); + } + std::printf("%-24s size=%zu/%zu cosine=%.9f maxdiff=%.6g rmse=%.6g %s\n", + name.c_str(), got.size(), want.size(), cosine(got, want), + max_abs_diff(got, want), std::sqrt(squared / std::max(1, got.size())), + ok ? "OK" : "FAIL"); + return ok; +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s [taps.npz] [vocos-path] [output.wav]\n", argv[0]); + return 2; + } + const auto arrays = load_npz(argv[1]); + // Arrays written by zipvoice/mlx/fixtures/build_fixture.py. Keyed by name: + // tokens, prompt_tokens, prompt_features(+lens), text_condition, x0, x1. + const auto find_arr = [&](const char * key) -> const NpyArray & { + for (const auto & a : arrays) { + if (a.name == key) return a; + } + std::fprintf(stderr, "fixture missing array '%s'\n", key); + std::exit(2); + }; + const auto & tokens_arr = find_arr("tokens"); + const auto & prompt_tokens_arr = find_arr("prompt_tokens"); + const auto & prompt_features = find_arr("prompt_features"); + const auto & prompt_features_lens = find_arr("prompt_features_lens"); + const auto & text_condition = find_arr("text_condition"); + const auto & x0 = find_arr("x0"); + const auto & x1 = find_arr("x1"); + + std::vector tokens(tokens_arr.data.size() / 4); + std::memcpy(tokens.data(), tokens_arr.data.data(), tokens_arr.data.size()); + std::vector prompt_tokens(prompt_tokens_arr.data.size() / 4); + std::memcpy(prompt_tokens.data(), prompt_tokens_arr.data.data(), prompt_tokens_arr.data.size()); + int64_t prompt_len = 0; + std::memcpy(&prompt_len, prompt_features_lens.data.data(), sizeof(int32_t)); + + const auto to_floats = [](const NpyArray & a) { + std::vector v(a.data.size() / 4); + std::memcpy(v.data(), a.data.data(), a.data.size()); + return v; + }; + + const std::string model = argv[2]; + std::optional bundle; + if (std::filesystem::path(model).extension() == ".gguf") { + bundle = engine::model_spec::load_resource_bundle_for_family(model, "zipvoice"); + } + const auto * resources = bundle ? &*bundle : nullptr; + bool checks_ok = true; + engine::models::zipvoice::ZipVoiceComputeDevice device; + device.backend_type = engine::core::BackendType::Cpu; // parity: bit-stable reference path + device.threads = 8; + + // ---- stage 0: raw text encoder vs torch taps (when provided) ---- + if (argc > 3) { + const auto taps = load_npz(argv[3]); + const auto find_tap_opt = [&](const char * key) -> const NpyArray * { + for (const auto & a : taps) { + if (a.name == key) return &a; + } + return nullptr; + }; + // torch embed is [1, S+1, feat]; row-major == [S+1, F] rows (optional: + // the layer0 tap file lacks it) + const NpyArray * embed_ref = find_tap_opt("embed"); + // reference runs the encoder on prompt_tokens + tokens (pad appended + // inside the hook) + std::vector cat_ids(prompt_tokens); + cat_ids.insert(cat_ids.end(), tokens.begin(), tokens.end()); + engine::models::zipvoice::ZipVoiceLayerTaps got_taps; + const auto got_embed = engine::models::zipvoice::zipvoice_text_encoder_raw( + model, cat_ids, &got_taps, device, resources); + if (embed_ref != nullptr) { + const auto want_embed = to_floats(*embed_ref); + checks_ok &= check("text_encoder_raw", got_embed, want_embed, 2e-6F, 2e-5F); + } + // per-layer outputs (torch (S+1, 1, C) rows) for bisecting + for (size_t li = 0; li < got_taps.layers.size(); ++li) { + const std::string name = "layer" + std::to_string(li); + const NpyArray * ref = find_tap_opt(name.c_str()); + if (ref == nullptr) continue; + const auto want = to_floats(*ref); + const auto & got = got_taps.layers[li]; + checks_ok &= check(name, got, want, 3e-5F, 2e-5F); + } + const size_t stage_count = got_taps.stages.size(); + const char * got_names[] = { + "layer_in", "attn_qtb", "attn_ktb", "attn_inproj", "attn_scores", + "attn_pos", "attn_w", "ff1_inproj", "ff1_act", "ff1", "na", + "na_gated", "na_gtb", "na_attnrep", "na_attended", "na_mul", "na_y2", "na_h", "sa1", + "conv1", "ff2", "bypass_mid", "sa2", "conv2", "ff3", "norm"}; + if (stage_count != sizeof(got_names) / sizeof(got_names[0])) { + throw std::runtime_error("text stage tap/name count mismatch"); + } + for (size_t si = 0; si < stage_count; ++si) { + const auto & got = got_taps.stages[si]; + const char * ref_key = nullptr; + // map got -> ref by semantic name + const char * gn = got_names[si]; + if (std::strcmp(gn, "layer_in") == 0) ref_key = "layer_in"; + else if (std::strcmp(gn, "attn_qtb") == 0) ref_key = "attn_qtb_ref"; + else if (std::strcmp(gn, "attn_ktb") == 0) ref_key = "attn_ktb_ref"; + else if (std::strcmp(gn, "attn_inproj") == 0) ref_key = "attn_inproj_ref"; + else if (std::strcmp(gn, "attn_scores") == 0) ref_key = "attn_scores_ref"; + else if (std::strcmp(gn, "attn_pos") == 0) ref_key = "attn_pos_ref"; + else if (std::strcmp(gn, "attn_w") == 0) ref_key = "self_attn_weights"; + else if (std::strcmp(gn, "ff1_inproj") == 0) ref_key = "ff1_inproj"; + else if (std::strcmp(gn, "ff1_act") == 0) ref_key = "ff1_act"; + else if (std::strcmp(gn, "ff1") == 0) ref_key = "feed_forward1"; + else if (std::strcmp(gn, "na") == 0) ref_key = "nonlin_attention"; + else if (std::strcmp(gn, "na_gated") == 0) ref_key = "na_gated_ref"; + else if (std::strcmp(gn, "na_gtb") == 0) ref_key = "na_gtb_ref"; + else if (std::strcmp(gn, "na_mul") == 0) ref_key = "na_mul_ref"; + else if (std::strcmp(gn, "na_y2") == 0) ref_key = "na_y2_ref"; + else if (std::strcmp(gn, "na_h") == 0) ref_key = "na_h_ref"; + else if (std::strcmp(gn, "na_attnrep") == 0) ref_key = "self_attn_weights"; + else if (std::strcmp(gn, "na_attended") == 0) ref_key = "na_attended_ref"; + else if (std::strcmp(gn, "sa1") == 0) ref_key = "self_attn1"; + else if (std::strcmp(gn, "conv1") == 0) ref_key = "conv_module1"; + else if (std::strcmp(gn, "ff2") == 0) ref_key = "feed_forward2"; + else if (std::strcmp(gn, "bypass_mid") == 0) ref_key = "bypass_mid"; + else if (std::strcmp(gn, "sa2") == 0) ref_key = "self_attn2"; + else if (std::strcmp(gn, "conv2") == 0) ref_key = "conv_module2"; + else if (std::strcmp(gn, "ff3") == 0) ref_key = "feed_forward3"; + else if (std::strcmp(gn, "norm") == 0) ref_key = "norm"; + if (ref_key == nullptr) continue; + // torch attention weights are (H, B, T, T); got is [S*T? ] handled below + if (getenv("ZV_DUMP") != nullptr) { + const std::string path = std::string("/tmp/zv_mine_") + gn + ".bin"; + std::ofstream f(path, std::ios::binary); + f.write(reinterpret_cast(got.data()), + static_cast(got.size() * sizeof(float))); + } + const NpyArray * ref = find_tap_opt(ref_key); + if (ref == nullptr) continue; + auto want = to_floats(*ref); + if ((std::strcmp(gn, "attn_qtb") == 0 || std::strcmp(gn, "attn_ktb") == 0) && ref->shape.size() == 3) { + const int64_t D = ref->shape[0], T = ref->shape[1], H = ref->shape[2]; + auto reordered = want; + for (int64_t d = 0; d < D; ++d) + for (int64_t t = 0; t < T; ++t) + for (int64_t h = 0; h < H; ++h) + reordered[d + D * (t + T * h)] = want[h + H * (t + T * d)]; + want = std::move(reordered); + } + if (std::strcmp(gn, "na_attended") == 0 && ref->shape.size() == 4) { + const int64_t H = ref->shape[0], B = ref->shape[1], T = ref->shape[2], D = ref->shape[3]; + auto reordered = want; + for (int64_t h = 0; h < H; ++h) + for (int64_t b = 0; b < B; ++b) + for (int64_t t = 0; t < T; ++t) + for (int64_t d = 0; d < D; ++d) + reordered[t + T * (d + D * (h + H * b))] = want[d + D * (t + T * (b + B * h))]; + want = std::move(reordered); + } + if (std::strcmp(gn, "na_attnrep") == 0 && ref->shape.size() == 4 && ref->shape[1] == 1) { + want.resize(static_cast(ref->shape[2] * ref->shape[3])); + } + // attention weights need a 4D reorder: torch (H, B, tgt, src) + // row-major vs ggml [src (ne0), tgt, H, B] + if (std::strcmp(gn, "attn_w") == 0) { + const int64_t H = ref->shape[0], B = ref->shape[1]; + const int64_t TT = ref->shape[2], TS = ref->shape[3]; + std::vector reordered(want.size(), 0.0F); + for (int64_t h = 0; h < H; ++h) + for (int64_t b = 0; b < B; ++b) + for (int64_t tt = 0; tt < TT; ++tt) + for (int64_t ts = 0; ts < TS; ++ts) { + const size_t torch_flat = static_cast( + h * B * TT * TS + b * TT * TS + tt * TS + ts); + const size_t mine_flat = static_cast( + ts + tt * TS + h * TS * TT + b * TS * TT * H); + reordered[mine_flat] = want[torch_flat]; + } + want = std::move(reordered); + } + if (got.size() != want.size()) { + std::printf(" stage %-18s: size cpp=%zu ref=%zu SKIP\n", gn, got.size(), want.size()); + checks_ok = false; + continue; + } + checks_ok &= check(gn, got, want, 3e-5F, 2e-5F); + + } + } + + // ---- stage 1: text_condition ---- + const auto want_text = to_floats(text_condition); + const auto got_text = engine::models::zipvoice::zipvoice_text_condition( + model, tokens, prompt_tokens, prompt_len, 1.0F, device, resources); + std::printf("text_condition: cpp=%zu ref=%zu cosine=%.6f maxdiff=%.4g\n", + got_text.size(), want_text.size(), + cosine(got_text, want_text), max_abs_diff(got_text, want_text)); + + // ---- stage 2: full sample (distill, 8 steps, guidance 3.0, t_shift 0.5) ---- + const auto prompt_feats = to_floats(prompt_features); + const auto want_x1 = to_floats(x1); + const auto got_x1 = engine::models::zipvoice::zipvoice_sample( + model, tokens, prompt_tokens, prompt_feats, prompt_len, + to_floats(x0), 8, 3.0F, 0.5F, 1.0F, device, resources); + std::printf("x1: cpp=%zu ref=%zu cosine=%.6f maxdiff=%.4g\n", + got_x1.size(), want_x1.size(), + cosine(got_x1, want_x1), max_abs_diff(got_x1, want_x1)); + + const bool text_ok = check("text_condition", got_text, want_text, 2e-6F, 2e-5F); + const bool x1_ok = check("sample", got_x1, want_x1, 3e-4F, 2e-4F); + for (const auto & array : arrays) { + if (array.name.rfind("velocity_", 0) != 0) continue; + int length = 0, step = 0; + if (std::sscanf(array.name.c_str(), "velocity_%d_%d", &length, &step) != 2) return 2; + auto noise = to_floats(x0); + auto text = want_text; + auto speech = to_floats(find_arr("speech_condition")); + noise.resize(length * 100); + text.resize(length * 100); + speech.resize(length * 100); + const auto velocity = engine::models::zipvoice::zipvoice_velocity( + model, noise, text, speech, length, step == 0 ? 0.0F : 0.6F, 3.0F, device, 1, resources); + checks_ok &= check(array.name, velocity, to_floats(array), 3e-4F, 2e-4F); + } + for (const auto & array : arrays) { + if (array.name == "batch_velocity") { + checks_ok &= check("batch_velocity", engine::models::zipvoice::zipvoice_velocity( + model, to_floats(find_arr("batch_x")), to_floats(find_arr("batch_text")), + to_floats(find_arr("batch_speech")), array.shape[1], 0.6F, 3.0F, device, 2, resources), + to_floats(array), 3e-4F, 2e-4F); + } + if (array.name == "fbank_mel") { + checks_ok &= check("fbank", engine::models::zipvoice::zipvoice_logmel( + to_floats(find_arr("fbank_wav"))), to_floats(array), 5e-5F, 2e-5F); + } + if (array.name == "vocos_audio") { + const std::string vocos = argc > 4 ? argv[4] : model; + checks_ok &= check("vocos", engine::models::zipvoice::zipvoice_vocos_decode( + vocos, to_floats(find_arr("vocos_mel"))), to_floats(array), 2e-4F, 2e-3F); + } + } + std::printf("%s %s\n", text_ok ? "TEXT_CONDITION OK" : "TEXT_CONDITION FAIL", + x1_ok ? "SAMPLE OK" : "SAMPLE FAIL"); + if (argc > 5 && text_ok && x1_ok && checks_ok) { + std::vector mel(got_x1.begin() + prompt_len * 100, got_x1.end()); + for (float & value : mel) value /= 0.1F; + auto audio = engine::models::zipvoice::zipvoice_vocos_decode(argv[4], mel); + double energy = 0; + for (float value : audio) { + if (!std::isfinite(value)) throw std::runtime_error("non-finite audio sample"); + energy += double(value) * value; + } + if (audio.empty() || energy / audio.size() < 1e-8) throw std::runtime_error("silent synthesis output"); + engine::audio::write_pcm16_wav(argv[5], 24000, 1, audio); + std::printf("Wrote %s: %.3f seconds, RMS %.6f\n", argv[5], audio.size() / 24000.0, + std::sqrt(energy / audio.size())); + } + return (text_ok && x1_ok && checks_ok) ? 0 : 1; +} diff --git a/tests/zipvoice/zipvoice_zh_tokens_main.cpp b/tests/zipvoice/zipvoice_zh_tokens_main.cpp new file mode 100644 index 000000000..ea2956c2e --- /dev/null +++ b/tests/zipvoice/zipvoice_zh_tokens_main.cpp @@ -0,0 +1,107 @@ +// Token-level parity test for the zipvoice Chinese frontend (emilia mode): +// compares EmiliaTokenizer::encode against reference ids dumped from the +// upstream tokenizer by tests/zipvoice/zh_reference_ids.py. +// +// Usage: zipvoice_zh_tokens +#include "engine/community_models/zipvoice/emilia_tokenizer.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::string slurp(const std::string & path) { + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error("cannot open " + path); + return std::string(std::istreambuf_iterator(in), std::istreambuf_iterator()); +} + +// Minimal JSON parsing for the flat {"text": {"ids": [...], "tokens": [...]}} +// file (corpus keys contain no escaped quotes). +struct Case { std::string text; std::vector ids; }; + +std::vector parse_cases(const std::string & json) { + std::vector cases; + size_t pos = 0; + while (true) { + const auto ids_key = json.find("\"ids\"", pos); + if (ids_key == std::string::npos) break; + const auto colon = json.rfind(':', ids_key); + const auto key_end = json.rfind('"', colon); + const auto key_start = json.rfind('"', key_end - 1); + if (colon == std::string::npos || key_end == std::string::npos || + key_start == std::string::npos) break; + Case entry; + entry.text = json.substr(key_start + 1, key_end - key_start - 1); + const auto ids_open = json.find('[', ids_key); + const auto ids_close = json.find(']', ids_open); + std::string ids_raw = json.substr(ids_open + 1, ids_close - ids_open - 1); + size_t start = 0; + while (start < ids_raw.size()) { + const auto comma = ids_raw.find(',', start); + const auto piece = ids_raw.substr(start, + comma == std::string::npos ? std::string::npos : comma - start); + if (!piece.empty()) entry.ids.push_back(std::stoi(piece)); + if (comma == std::string::npos) break; + start = comma + 1; + } + cases.push_back(std::move(entry)); + pos = ids_close; + } + return cases; +} + +std::unordered_map load_vocab(const std::filesystem::path & dir) { + std::ifstream in(dir / "tokens.txt"); + if (!in) throw std::runtime_error("cannot open tokens.txt in " + dir.string()); + std::unordered_map vocab; + std::string line; + while (std::getline(in, line)) { + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + vocab.emplace(line.substr(0, tab), std::stoi(line.substr(tab + 1))); + } + return vocab; +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + using namespace engine::models::zipvoice; + const std::filesystem::path model_dir = argv[2]; + const auto vocab = load_vocab(model_dir); + EmiliaTokenizer tokenizer( + EmiliaTokenizer::TablePaths::from_model_dir(model_dir), vocab, + EmiliaTokenizer::EspeakConfig{ + "/opt/homebrew/lib/libespeak-ng.dylib", "", "en-us"}); + + const auto cases = parse_cases(slurp(argv[1])); + int failures = 0; + for (const auto & c : cases) { + const auto got = tokenizer.encode(c.text); + bool ok = got.size() == c.ids.size(); + for (size_t i = 0; ok && i < got.size(); ++i) ok = got[i] == c.ids[i]; + if (!ok) { + ++failures; + std::printf("FAIL %-30s got=%zu want=%zu\n", c.text.c_str(), got.size(), c.ids.size()); + std::printf(" got :"); + for (int32_t v : got) std::printf(" %d", v); + std::printf("\n want:"); + for (int32_t v : c.ids) std::printf(" %d", v); + std::printf("\n"); + } else { + std::printf("OK %s (%zu ids)\n", c.text.c_str(), got.size()); + } + } + std::printf("%d/%zu cases passed\n", static_cast(cases.size() - failures), cases.size()); + return failures == 0 ? 0 : 1; +} diff --git a/tools/community_models/convert_zipvoice.py b/tools/community_models/convert_zipvoice.py new file mode 100644 index 000000000..cfb372509 --- /dev/null +++ b/tools/community_models/convert_zipvoice.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Convert ZipVoice / ZipVoice-Distill torch checkpoints to audio.cpp packages. + +ZipVoice (https://github.com/k2-fsa/ZipVoice, k2-fsa/ZipVoice on HuggingFace) +ships `model.pt` checkpoints (a pickled dict with a "model" state dict), a +`model.json` architecture config, and a `tokens.txt` phone vocab. audio.cpp +cannot read the nested pickle directly, so this script first flattens the +state dict to safetensors (the development format) and then packages it into +a self-contained GGUF with two tensor namespaces: + + model.* — the ZipVoice flow-matching model (raw torch names) + vocos.* — the Vocos mel-24kHz vocoder used to decode generated features + +`tokens.txt` and `model.json` are copied next to the output so the GGUF plus +those two files form a complete model directory for +`audiocpp_cli --family zipvoice`. + +Examples: + # ZipVoice-Distill (8 steps, guidance-scale embedding) with bundled vocoder + python3 tools/community_models/convert_zipvoice.py \ + --model-dir /models/ZipVoice/zipvoice_distill \ + --vocos /models/vocos-mel-24khz/vocos.safetensors \ + --converter build/bin/audiocpp_gguf + + # safetensors only (development format, no GGUF step) + python3 tools/community_models/convert_zipvoice.py \ + --model-dir /models/ZipVoice/zipvoice_distill --safetensors-only +""" +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import torch + + +def load_state_dict(model_dir: Path, checkpoint_name: str) -> dict: + ckpt_path = model_dir / checkpoint_name + if not ckpt_path.is_file(): + raise SystemExit(f"checkpoint not found: {ckpt_path}") + ckpt = torch.load(str(ckpt_path), map_location="cpu", weights_only=False) + if isinstance(ckpt, dict) and "model" in ckpt: + state = ckpt["model"] + else: + state = ckpt + if not isinstance(state, dict): + raise SystemExit(f"unexpected checkpoint layout in {ckpt_path}") + return {name: tensor.contiguous().to(torch.float32) + for name, tensor in state.items() + if isinstance(tensor, torch.Tensor)} + + +def write_safetensors(state: dict, output: Path) -> None: + from safetensors.torch import save_file + output.parent.mkdir(parents=True, exist_ok=True) + save_file(state, str(output)) + print(f"wrote {output} ({output.stat().st_size / 1e6:.1f} MB, " + f"{len(state)} tensors)") + + +def convert_gguf(converter: Path, safetensors: Path, vocos: Path, + output: Path, quant_type: str, overwrite: bool, + root: Path) -> None: + command = [ + str(converter), + "--input", f"model={safetensors}", + "--input", f"vocos={vocos}", + "--root", str(root), + "--family", "zipvoice", + "--type", quant_type, + "--output", str(output), + ] + if overwrite: + command.append("--overwrite") + print("+", " ".join(command)) + subprocess.run(command, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--model-dir", type=Path, required=True, + help="ZipVoice model dir with model.pt / model.json / tokens.txt") + parser.add_argument("--checkpoint-name", default="model.pt") + parser.add_argument("--vocos", type=Path, + default=Path("/models/vocos-mel-24khz/vocos.safetensors"), + help="vocos.safetensors (bundled into the GGUF)") + parser.add_argument("--converter", type=Path, + default=Path("build/bin/audiocpp_gguf")) + parser.add_argument("--output-dir", type=Path, default=None, + help="default: /converted") + parser.add_argument("--type", default="orig", + choices=["orig", "f16", "bf16", "q8_0", "q2_k", "q3_k", + "q4_k", "q5_k", "q6_k"], + help="GGUF storage type (default orig = keep f32)") + parser.add_argument("--name", default=None, + help="package base name (default: model-dir name, lowercased)") + parser.add_argument("--safetensors-only", action="store_true", + help="stop after writing the flattened safetensors") + parser.add_argument("--no-vocos", action="store_true", + help="do not bundle a vocoder namespace into the GGUF") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + model_dir = args.model_dir.resolve() + for required in ("model.json", "tokens.txt"): + if not (model_dir / required).is_file(): + raise SystemExit(f"{required} missing in {model_dir}") + with (model_dir / "model.json").open() as f: + config = json.load(f) + has_guidance_embed = ( + config.get("distill", {}).get("guidance_scale_embed", False) + if isinstance(config.get("distill"), dict) else False + ) + name = args.name or model_dir.name.lower().replace("_", "-") + out_dir = (args.output_dir or model_dir / "converted").resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + state = load_state_dict(model_dir, args.checkpoint_name) + print(f"loaded {len(state)} tensors from {model_dir / args.checkpoint_name}") + + # Fixed name matching model_specs/zipvoice.json sources[format=safetensors] + # ("model:zipvoice-orig.safetensors"); the GGUF keeps the - naming. + safetensors_path = out_dir / "zipvoice-orig.safetensors" + write_safetensors(state, safetensors_path) + + # dev-format directory: safetensors + config + vocab + shutil.copyfile(model_dir / "tokens.txt", out_dir / "tokens.txt") + shutil.copyfile(model_dir / "model.json", out_dir / "model.json") + # Chinese frontend sidecars (pypinyin tables + jieba dictionaries), when + # the model directory was staged by export_zipvoice_zh_dict.py. Staged + # into out_dir BEFORE packaging: the converter root is scanned for + # sidecars, so these ride inside the GGUF and a converted package is a + # single self-sufficient file (loose copies below remain for the + # directory layout). + for sidecar in sorted(model_dir.glob("zh_*")): + shutil.copyfile(sidecar, out_dir / sidecar.name) + staged_zh = sorted(p.name for p in out_dir.glob("zh_*")) + print(f"copied tokens.txt / model.json -> {out_dir}" + + (f" + {len(staged_zh)} zh frontend sidecars" if staged_zh else "")) + + if args.safetensors_only: + return + + converter = args.converter.resolve() + if not converter.is_file(): + raise SystemExit(f"converter not found: {converter} (build target audiocpp_gguf)") + + gguf_dir = out_dir / name + gguf_dir.mkdir(parents=True, exist_ok=True) + gguf_path = gguf_dir / f"{name}-{args.type}.gguf" + # NOTE: --root must point at a directory with real (non-symlinked) + # sidecar files; HF-cache snapshots use blob symlinks that the sidecar + # scanner does not enumerate, so pass the flattened dev directory that + # already carries tokens.txt / model.json copies. + root = out_dir if (out_dir / "tokens.txt").is_file() else model_dir + if args.no_vocos: + command = [ + str(converter), + "--input", f"model={safetensors_path}", + "--root", str(root), + "--family", "zipvoice", + "--type", args.type, + "--output", str(gguf_path), + ] + if args.overwrite: + command.append("--overwrite") + print("+", " ".join(command)) + subprocess.run(command, check=True) + else: + vocos = args.vocos.resolve() + if not vocos.is_file(): + raise SystemExit(f"vocos checkpoint not found: {vocos} " + "(pass --no-vocos or --vocos )") + convert_gguf(converter, safetensors_path, vocos, gguf_path, + args.type, args.overwrite, root) + shutil.copyfile(model_dir / "tokens.txt", gguf_dir / "tokens.txt") + shutil.copyfile(model_dir / "model.json", gguf_dir / "model.json") + # Chinese frontend sidecars (pypinyin tables + jieba dictionaries), when + # the model directory was staged by export_zipvoice_zh_dict.py. + for sidecar in sorted(model_dir.glob("zh_*")): + shutil.copyfile(sidecar, gguf_dir / sidecar.name) + print(f"package ready: {gguf_path}") + + +if __name__ == "__main__": + main() diff --git a/tools/community_models/export_zipvoice_zh_dict.py b/tools/community_models/export_zipvoice_zh_dict.py new file mode 100644 index 000000000..58e072ae0 --- /dev/null +++ b/tools/community_models/export_zipvoice_zh_dict.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Export Chinese G2P dictionaries for the ZipVoice audio.cpp frontend. + +The upstream ZipVoice EmiliaTokenizer converts Chinese text with +jieba + pypinyin (TONE3 style, tone sandhi, neutral tone = 5) and splits +every syllable into an initial token (e.g. "w0") plus a final token +(e.g. "o3"). audio.cpp implements the lookup in C++ but must reproduce +pypinyin's syllable splitting exactly, so this script bakes the split +into the exported tables: + + zh_chars.tsv \\t (default reading) + zh_phrases.tsv \\t (raw tones, no sandhi) + zh_syllables.tsv \\t (for tags) + +Tone sandhi (3-3 runs, 一/不) is applied by the C++ frontend at run time, +mirroring lazy_pinyin(tone_sandhi=True); the tables therefore stay raw. + +This script also stages the jieba segmentation dictionaries used by the +C++ frontend (the model-local JiebaSegmenter, equivalent to python +jieba.cut with HMM on): + + zh_jieba_dict.txt lines (jieba jieba.dict.utf8) + zh_hmm_model.txt HMM start/trans/emit tables (jieba hmm_model.utf8) + +The dictionaries are not vendored in this repository; they are downloaded +from the cppjieba mirror commit pinned below (SHA-256 verified, cached +under ~/.cache/audiocpp) so the packaged model is self-contained. Pass +--cppjieba-dict to copy from a local cppjieba checkout instead +(offline escape hatch). + +Splitting follows pypinyin's to_initials(strict=False) + +to_finals_tone3(strict=False, neutral_tone_with_five=True): y/w ARE +initials ("yi3" -> "y0 i3"), and ju/qu/xu keep "u" (only nv/lv give "v"). + +Run with the same Python environment used for the upstream checkout, e.g. + /path/to/ZipVoice/.venv/bin/python \\ + tools/community_models/export_zipvoice_zh_dict.py \\ + --output-dir /models/ZipVoice/zipvoice_distill +""" +import argparse +import hashlib +import unicodedata +import urllib.request +from pathlib import Path + +# These jieba dictionaries are pinned to a cppjieba mirror commit (cppjieba +# redistributes the jieba dictionaries); byte-for-byte identity is enforced by +# SHA-256 so exported packages stay reproducible. +CPPJIEBA_COMMIT = "8f171de" +CPPJIEBA_DICT_URL = ("https://raw.githubusercontent.com/yanyiwu/cppjieba/" + + CPPJIEBA_COMMIT + "/dict/{name}") +CPPJIEBA_DICT_SHA256 = { + "jieba.dict.utf8": + "6f7d4350e8861ef4139b2e3a6fad05430c19ae71f4b8378190edecac8aae2e6a", + "hmm_model.utf8": + "f17790586ac86dd048c8adffed052c4bd2b28ed0682972c1275e59040c0589a7", +} +DEFAULT_DICT_CACHE_DIR = Path.home() / ".cache" / "audiocpp" / "cppjieba-dict" + +_TONE_MARKS = { + "ā": ("a", 1), "á": ("a", 2), "ǎ": ("a", 3), "à": ("a", 4), + "ē": ("e", 1), "é": ("e", 2), "ě": ("e", 3), "è": ("e", 4), + "ī": ("i", 1), "í": ("i", 2), "ǐ": ("i", 3), "ì": ("i", 4), + "ō": ("o", 1), "ó": ("o", 2), "ǒ": ("o", 3), "ò": ("o", 4), + "ū": ("u", 1), "ú": ("u", 2), "ǔ": ("u", 3), "ù": ("u", 4), + "ǖ": ("v", 1), "ǘ": ("v", 2), "ǚ": ("v", 3), "ǜ": ("v", 4), + "ń": ("n", 2), "ň": ("n", 3), "ǹ": ("n", 4), + "ḿ": ("m", 2), +} + +_INITIALS = [ + "zh", "ch", "sh", "b", "p", "m", "f", "d", "t", "n", "l", "g", "k", + "h", "j", "q", "x", "r", "z", "c", "s", "y", "w", +] + + +def to_tone3(syllable: str) -> str: + """Convert an accented pinyin syllable (e.g. "xiǎo") to TONE3 ("xiao3").""" + syllable = unicodedata.normalize("NFKC", syllable) + out = [] + tone = 0 + i = 0 + while i < len(syllable): + ch = syllable[i] + two = syllable[i:i + 2] + if two in _TONE_MARKS: + base, t = _TONE_MARKS[two] + out.append(base) + tone = t + i += 2 + continue + if ch in _TONE_MARKS: + base, t = _TONE_MARKS[ch] + out.append(base) + tone = t + else: + out.append(ch) + i += 1 + if tone == 0: + tone = 5 # neutral-tone syllables carry no mark + return "".join(out) + str(tone) + + +def split_syllable(tone3: str) -> list[str]: + """Split a TONE3 syllable into initial ("X0") + final ("Ytone") tokens.""" + tone = tone3[-1] + body = tone3[:-1] + if not body.isalpha(): + return [tone3] + initial = "" + rest = body + for cand in _INITIALS: + if body.startswith(cand): + initial = cand + rest = body[len(cand):] + break + tokens = [] + if initial: + tokens.append(initial + "0") + if rest: + tokens.append(rest + tone) + return tokens + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def stage_jieba_dicts(output_dir: Path, cache_dir: Path, + override_dir: Path | None) -> None: + """Write zh_jieba_dict.txt / zh_hmm_model.txt next to tokens.txt. + + Downloads the pinned upstream dictionaries into cache_dir on first use. + --cppjieba-dict overrides the download with a local cppjieba dict/ + directory (copied as-is; its hashes are printed for visibility). + """ + for name in ("jieba.dict.utf8", "hmm_model.utf8"): + target = output_dir / ("zh_hmm_model.txt" if name.endswith("hmm_model.utf8") + else "zh_jieba_dict.txt") + if override_dir is not None: + source = override_dir / name + if not source.is_file(): + raise SystemExit(f"--cppjieba-dict override missing {name}: {source}") + data = source.read_bytes() + note = f"copied from {source} (sha256 {_sha256(data)})" + else: + cache = cache_dir / name + if cache.is_file() and _sha256(cache.read_bytes()) == CPPJIEBA_DICT_SHA256[name]: + data = cache.read_bytes() + note = f"cached {cache}" + else: + url = CPPJIEBA_DICT_URL.format(name=name) + print(f"downloading {url}") + with urllib.request.urlopen(url, timeout=120) as response: + data = response.read() + digest = _sha256(data) + if digest != CPPJIEBA_DICT_SHA256[name]: + raise SystemExit( + f"downloaded {name} SHA-256 {digest} does not match the pinned " + f"{CPPJIEBA_DICT_SHA256[name]}; pass --cppjieba-dict to copy the " + "dictionaries from a local cppjieba checkout instead") + cache_dir.mkdir(parents=True, exist_ok=True) + cache.write_bytes(data) + note = f"downloaded (pinned {CPPJIEBA_COMMIT}, SHA-256 verified)" + target.write_bytes(data) + print(f"staged {name} -> {target} [{note}]") + + +def export_pinyin_tables(output_dir: Path) -> None: + # Deferred import: the dictionary staging above only needs the standard + # library, so the download path runs in any Python; only the table baking + # requires the upstream ZipVoice environment. + try: + from pypinyin.phrases_dict import phrases_dict + from pypinyin.pinyin_dict import pinyin_dict + except ImportError as error: + raise SystemExit( + "pypinyin is required to bake the pronunciation tables; run this " + "script with the upstream ZipVoice python environment " + f"(import failed: {error})") from error + + # Characters: pinyin_dict maps codepoint -> space-separated accented + # pinyins; the first entry is pypinyin's default reading. + bare_syllables = set() + chars_path = output_dir / "zh_chars.tsv" + with chars_path.open("w", encoding="utf-8") as f: + for codepoint, pinyins in sorted(pinyin_dict.items()): + if not (0x3400 <= codepoint <= 0x9FFF or 0xF900 <= codepoint <= 0xFAFF): + continue + # Entries are comma-separated readings (e.g. "hǔ,hù"); the + # first is pypinyin's default reading. + first = to_tone3(pinyins.split(",")[0].strip()) + bare_syllables.add(first) + tokens = split_syllable(first) + f.write(f"{chr(codepoint)}\t{' '.join(tokens)}\n") + print(f"wrote {chars_path}") + + # Phrases: raw per-syllable tones (no sandhi); longest-match lookup and + # tone sandhi happen in C++. + phrases_path = output_dir / "zh_phrases.tsv" + count = 0 + with phrases_path.open("w", encoding="utf-8") as f: + for phrase, syllables in sorted(phrases_dict.items()): + if not all(len(s) == 1 for s in syllables): + continue + tokens = [] + for entry in syllables: + tokens.extend(split_syllable(to_tone3(entry[0]))) + f.write(f"{phrase}\t{' '.join(tokens)}\n") + count += 1 + print(f"wrote {phrases_path} ({count} entries)") + + # Valid bare tone3 syllables for overrides. + syllables_path = output_dir / "zh_syllables.tsv" + with syllables_path.open("w", encoding="utf-8") as f: + for syllable in sorted(bare_syllables): + f.write(f"{syllable}\t{' '.join(split_syllable(syllable))}\n") + print(f"wrote {syllables_path} ({len(bare_syllables)} entries)") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True, + help="model directory that contains tokens.txt") + parser.add_argument("--cppjieba-dict", type=Path, default=None, + help="offline override: cppjieba dict/ directory to copy " + "from (default: download the pinned upstream " + "dictionaries, SHA-256 verified)") + parser.add_argument("--dict-cache-dir", type=Path, default=DEFAULT_DICT_CACHE_DIR, + help="download cache directory (default: %(default)s)") + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + + # jieba segmentation dictionaries for the model-local segmenter. + stage_jieba_dicts(args.output_dir, args.dict_cache_dir, args.cppjieba_dict) + + export_pinyin_tables(args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json index 8a43aa208..f463efb11 100644 --- a/webui/configs/model_params.json +++ b/webui/configs/model_params.json @@ -340,6 +340,14 @@ {"name": "duration_factor", "type": "slider", "label": "duration_factor（语速/时长倍率，>1 更慢，<1 更快）", "label_en": "duration_factor (duration multiplier; >1 slower, <1 faster)", "default": 1.0, "minimum": 0.5, "maximum": 2.0, "step": 0.05, "info": "对齐官方 IndexTTS2.5 的 duration_factor：缩放输出时长，不改变音色/内容", "info_en": "Matches official IndexTTS2.5 duration_factor: scales output duration without changing timbre or content"} ], + "zipvoice": [ + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale（引导强度，0=关闭）", "label_en": "guidance_scale (0 disables CFG)", "default": 3.0, "minimum": 0.0, "maximum": 10.0, "step": 0.5}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（Euler 步数）", "label_en": "num_inference_steps", "default": 8, "minimum": 1, "maximum": 64, "step": 1, "precision": 0}, + {"name": "t_shift", "type": "slider", "label": "t_shift（时间步偏移，越小越偏低 SNR）", "label_en": "t_shift (timestep shift)", "default": 0.5, "minimum": 0.05, "maximum": 1.0, "step": 0.05}, + {"name": "speed", "type": "slider", "label": "speed（语速倍率）", "label_en": "speed (duration multiplier)", "default": 1.0, "minimum": 0.5, "maximum": 2.0, "step": 0.05}, + {"name": "lang", "type": "text", "label": "lang（英文段 espeak 语言）", "label_en": "lang (espeak voice for English runs)", "default": "en-us", "placeholder": "en-us"} + ], + "irodori_tts": [ {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（RF 扩散步数）", "label_en": "num_inference_steps", "default": 40, "minimum": 1, "step": 1, "precision": 0}, {"name": "duration_sec", "type": "number", "label": "duration_sec（0=模型自动预测时长）", "label_en": "duration_sec (0 = auto)", "default": 0, "minimum": 0, "step": 0.5}, diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 7d557dfae..271811570 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -83,6 +83,9 @@ { "id": "echo-tts", "display_name": "Echo-TTS (voice clone)", "family": "echo_tts", "path": "models/Echo-TTS-GGUF", "task": "clon", "mode": "offline", "download_id": "echo_tts_q8_0", "min_vram_gb": 8, "input_hint_en": "**Echo-TTS**: English zero-shot cloning at 44.1 kHz. Upload a reference voice -- no transcript needed. Output is CC-BY-NC-SA and may not be used commercially." }, + { "id": "zipvoice", "display_name": "ZipVoice (中英零样本克隆)", "display_name_en": "ZipVoice (zh/en zero-shot cloning)", "family": "zipvoice", "path": "models/ZipVoice-Distill-GGUF", "task": "clon", "mode": "offline", "download_id": "zipvoice_distill_gguf", "min_vram_gb": 4, + "input_hint": "**ZipVoice**：中/英零样本声音克隆（k2-fsa TTSZipformer flow-matching + Vocos，蒸馏版 8 步采样）。上传参考音频，并在『参考文本’里填入它的逐字转写——参考音频与参考文本必须配对；合成文本支持中文、英文及混排。权重 Apache-2.0。", + "input_hint_en": "**ZipVoice**: zero-shot voice cloning in zh/en (k2-fsa TTSZipformer flow matching + Vocos, distilled 8-step sampling). Upload a reference clip and put its exact transcript in the reference-text box — the clip and the transcript must match; synthesis text supports zh/en/mixed. Weights are Apache-2.0." }, { "id": "chatterbox", "display_name": "Chatterbox (voice clone)", "family": "chatterbox", "path": "models/chatterbox", "task": "clon", "mode": "offline", "download_id": "chatterbox", "min_vram_gb": 12 }, { "id": "chatterbox-turbo", "display_name": "Chatterbox Turbo (tts)", "family": "chatterbox_turbo", "path": "models/Chatterbox-Turbo-GGUF/chatterbox-turbo-q8_0.gguf", "task": "tts", "mode": "offline", "download_id": "chatterbox_turbo_q8_0", "min_vram_gb": 4, "input_hint_en": "**Chatterbox Turbo**: fast English TTS with the built-in voice. No reference voice is required." }, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 189472bcf..23cc52eeb 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@