From 7432a425492979fdfcd312138c03ae1a391629f1 Mon Sep 17 00:00:00 2001 From: David Feng Date: Fri, 18 Sep 2026 18:07:01 +0900 Subject: [PATCH 1/4] zipvoice: add k2-fsa ZipVoice community TTS with zh/en frontend Zero-shot voice cloning on a TTSZipformer flow-matching backbone with a Vocos mel-24kHz vocoder; supports the distilled 8-step variant (guidance-scale embedding) and the base model (batched CFG). - TTSZipformer ggml graphs: text encoder with duration-ratio conditioning, U-Net flow-matching decoder (relative-position attention, non-linear attention, downsampling stacks) in the physical [C,T,B] convention, built through the framework module wrappers (Linear/Softmax/ DepthwiseConv1d/Embedding/BiasNorm/...) with raw strided views and the batch-broadcast mul_mat fallback where no wrapper exists - GPU-first runtime: ZipVoiceComputeDevice resolves BestAvailable to Metal/CUDA with CPU fallback for the whole stack (text encoder, flow-matching decoder, Vocos backbone); extend the ggml Metal F32 tiled matmul kernel to short contractions (32 <= K < 64, M >= 64, N >= 32, explicit GGML_PREC_F32) via a dedicated F32-operand pipeline plus a padded-row guard, so ZipVoice's 32-dim QK products leave the matrix-vector path and Distill reaches RTF < 1 on Apple M3 - Vocos vocoder through the shared framework ConvNeXt/iSTFT backbone; single self-contained GGUF packaging: model.* + vocos.* namespaces with the tokens/config and the zh frontend sidecars embedded, so the hosted package is one file (davidxifeng/zipvoice-gguf, huggingface_snapshot download); the model spec is GGUF-only and the naming follows the repo convention (-.gguf inside a -GGUF directory) - Chinese/English text frontend (tokenizer=emilia) reproducing the upstream EmiliaTokenizer: framework Chinese normalization, punctuation mapping, jieba segmentation via a model-local JiebaSegmenter (PreFilter separator pass, maximum-probability DAG over jieba.dict.utf8, four-state BMES HMM from hmm_model.utf8; equivalent to cppjieba::MixSegment, with MIT attribution to cppjieba (c) 2013 Yanyi Wu and jieba (c) 2012 Sun Junyi), pypinyin readings from baked zh_chars/zh_phrases/zh_syllables sidecars, per-word tone sandhi, espeak-ng for English runs; tokenizer=espeak|simple kept as alternatives. The segmenter dictionaries are model resources embedded in the GGUF (zh_jieba_dict.txt / zh_hmm_model.txt); export_zipvoice_zh_dict.py documents the pinned download source (URL/SHA-256 unchanged) - Session-owned runtime: weights, Vocos vocoder and bounded runtime::CacheSlots for the text-encoder and flow-decoder graphs live on the per-device ZipVoiceRuntimeState, so zipvoice_clear_runtime releases their memory and no process-global cache exists (RAII ZipVoiceGraphResources frees graph contexts, backend buffer and gallocr); long text is chunked with runtime::chunk_text_request and per-chunk audio merged via runtime::append_audio_buffer; reference audio is mixed to mono and resampled via convert_interleaved_audio_to_mono_torchaudio_sinc_hann_resampled with ref_channels on the synthesis request; the session loads the spec-resolved resource bundle so embedded sidecars materialize and resolve from the downloaded file alone, while direct API callers (parity harnesses) fall back to loose files next to the checkpoint - Conversion tools: convert_zipvoice.py (torch -> safetensors -> GGUF, staging the frontend sidecars into the converter root) and export_zipvoice_zh_dict.py - Verification: full-stage parity against the reference PyTorch implementation (encoder taps, text conditioning, sampled features, single+batched velocity, fbank, vocoder audio), token-for-token frontend parity against the upstream tokenizer, byte-identical segmentation against cppjieba::MixSegment over 349,011 dictionary entries plus 20,000 synthetic mixed cases, CPU/Metal parity (cosine 1.0 across all taps), and end-to-end zh/en/mixed synthesis ASR-verified from the hosted package - Docs: docs/community_models/zipvoice.md, catalog and README rows --- CMakeLists.txt | 27 + README.md | 1 + docs/community_models/models.md | 1 + docs/community_models/zipvoice.md | 163 +++ docs/espeak_phonemizer.md | 6 +- .../ggml/src/ggml-metal/ggml-metal-device.cpp | 13 +- .../ggml/src/ggml-metal/ggml-metal-ops.cpp | 10 +- external/ggml/src/ggml-metal/ggml-metal.metal | 17 +- .../zipvoice/emilia_tokenizer.h | 80 ++ .../community_models/zipvoice/session.h | 12 + .../community_models/zipvoice/synthesize.h | 151 +++ .../community_models/zipvoice/weights.h | 111 ++ .../community_models/zipvoice/zipformer.h | 84 ++ .../framework/modules/convnext_modules.h | 18 + .../modules/vocoders/vocos_vocoder.h | 34 + model_specs/zipvoice.json | 221 ++++ .../zipvoice/emilia_tokenizer.cpp | 342 ++++++ .../zipvoice/jieba_segmenter.cpp | 390 +++++++ .../zipvoice/jieba_segmenter.h | 67 ++ src/community_models/zipvoice/session.cpp | 278 +++++ src/community_models/zipvoice/synthesize.cpp | 1030 +++++++++++++++++ src/community_models/zipvoice/weights.cpp | 255 ++++ src/community_models/zipvoice/zipformer.cpp | 931 +++++++++++++++ src/framework/audio/espeak_phonemizer.cpp | 26 + src/framework/modules/convnext_modules.cpp | 23 + .../modules/vocoders/vocos_vocoder.cpp | 143 +++ tests/zipvoice/build_reference.py | 119 ++ tests/zipvoice/zh_reference_ids.py | 58 + tests/zipvoice/zipvoice_parity_main.cpp | 408 +++++++ tests/zipvoice/zipvoice_zh_tokens_main.cpp | 107 ++ tools/community_models/convert_zipvoice.py | 189 +++ .../export_zipvoice_zh_dict.py | 241 ++++ webui/configs/model_params.json | 8 + webui/configs/models_catalog.json | 3 + webui/native/dist/index.html | 74 +- 35 files changed, 5593 insertions(+), 48 deletions(-) create mode 100644 docs/community_models/zipvoice.md create mode 100644 include/engine/community_models/zipvoice/emilia_tokenizer.h create mode 100644 include/engine/community_models/zipvoice/session.h create mode 100644 include/engine/community_models/zipvoice/synthesize.h create mode 100644 include/engine/community_models/zipvoice/weights.h create mode 100644 include/engine/community_models/zipvoice/zipformer.h create mode 100644 include/engine/framework/modules/convnext_modules.h create mode 100644 include/engine/framework/modules/vocoders/vocos_vocoder.h create mode 100644 model_specs/zipvoice.json create mode 100644 src/community_models/zipvoice/emilia_tokenizer.cpp create mode 100644 src/community_models/zipvoice/jieba_segmenter.cpp create mode 100644 src/community_models/zipvoice/jieba_segmenter.h create mode 100644 src/community_models/zipvoice/session.cpp create mode 100644 src/community_models/zipvoice/synthesize.cpp create mode 100644 src/community_models/zipvoice/weights.cpp create mode 100644 src/community_models/zipvoice/zipformer.cpp create mode 100644 src/framework/modules/convnext_modules.cpp create mode 100644 src/framework/modules/vocoders/vocos_vocoder.cpp create mode 100644 tests/zipvoice/build_reference.py create mode 100644 tests/zipvoice/zh_reference_ids.py create mode 100644 tests/zipvoice/zipvoice_parity_main.cpp create mode 100644 tests/zipvoice/zipvoice_zh_tokens_main.cpp create mode 100644 tools/community_models/convert_zipvoice.py create mode 100644 tools/community_models/export_zipvoice_zh_dict.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3fa2768fe..e28d392ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,8 @@ add_library(engine_core OBJECT src/framework/decoders/tdt_decoder_greedy_separate_heads.cpp src/framework/modules/linear_module.cpp src/framework/modules/packed_linear_weights.cpp + src/framework/modules/convnext_modules.cpp + src/framework/modules/vocoders/vocos_vocoder.cpp src/framework/modules/primitive_modules.cpp src/framework/modules/activation_modules.cpp src/framework/modules/norm_modules.cpp @@ -827,6 +829,21 @@ audiocpp_add_model(f5_tts habibi_tts ) +audiocpp_add_model(zipvoice + SOURCES + src/community_models/zipvoice/session.cpp + src/community_models/zipvoice/synthesize.cpp + src/community_models/zipvoice/emilia_tokenizer.cpp + src/community_models/zipvoice/jieba_segmenter.cpp + src/community_models/zipvoice/weights.cpp + src/community_models/zipvoice/zipformer.cpp + INCLUDES + engine/community_models/zipvoice/session.h + engine/community_models/zipvoice/synthesize.h + LOADERS + engine::models::zipvoice::make_zipvoice_loader +) + audiocpp_add_model(minimax_h3 SOURCES src/community_models/minimax_h3/assets.cpp @@ -2901,6 +2918,16 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST endif() endif() # Model-specific tests and probes. + if (zipvoice IN_LIST AUDIOCPP_LINKED_MODELS) + add_executable(zipvoice_parity tests/zipvoice/zipvoice_parity_main.cpp) + target_link_libraries(zipvoice_parity PRIVATE engine_runtime ggml) + target_include_directories(zipvoice_parity PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + if (ENGINE_ENABLE_OPENMP) + target_link_libraries(zipvoice_parity PRIVATE OpenMP::OpenMP_CXX) + endif() + add_executable(zipvoice_zh_tokens tests/zipvoice/zipvoice_zh_tokens_main.cpp) + target_link_libraries(zipvoice_zh_tokens PRIVATE engine_runtime) + endif() if (f5_tts IN_LIST AUDIOCPP_LINKED_MODELS) target_compile_definitions(engine_model_f5_tts PRIVATE F5_MEL_TEST=1) foreach(f5_test IN ITEMS f5_e2e f5_parity f5_cfg_parity f5_tokenizer) diff --git a/README.md b/README.md index cfc46696b..ac7f5f482 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ Community model ports live under `community_models` to make the ownership bounda | **echo_tts** | Clone | en | GGUF 16/Q8 | [@5uck1ess](https://github.com/5uck1ess) | [Echo-TTS](docs/community_models/echo_tts.md) 44.1 kHz zero-shot voice cloning with EchoDiT latents and Fish S1-DAC decoding | | **f5_tts** | TTS, Clone | en, ar (Habibi) | GGUF | [@tareko](https://github.com/tareko) | [F5-TTS](docs/community_models/f5_tts.md) flow-matching DiT synthesis and voice cloning, with Habibi Arabic aliases `habibi`/`habibi_tts` | | **glm_tts** | TTS, Clone | zh, en | GGUF | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](docs/community_models/glm_tts.md) zero-shot synthesis and voice cloning support | +| **zipvoice** | TTS, Clone | zh, en | GGUF F32 | Community | [ZipVoice](docs/community_models/zipvoice.md) k2-fsa TTSZipformer flow-matching zero-shot voice cloning with Vocos vocoder; jieba + pypinyin Chinese frontend | | **granite5asr** | ASR | en | GGUF Q8 | [@ampersandru](https://github.com/ampersandru) | [IBM Granite Speech 5.0 470M TurboCTC](docs/community_models/granite5asr.md) ultra-fast Conformer-CTC ASR with Shaw relative positional embeddings and ByteLevel BPE | | **inflect_v2** | TTS | en | GGUF FP32 | Jan [@JanWerder](https://github.com/JanWerder) | [Inflect Micro v2 and Nano v2](docs/community_models/inflect_v2.md) native offline synthesis | | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Safetensors, GGUF Q8 | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](docs/community_models/kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps | diff --git a/docs/community_models/models.md b/docs/community_models/models.md index cc4553412..41c5a0cd5 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -21,6 +21,7 @@ Practical expectations: | **echo_tts** | TTS, voice cloning | en | Tym [@5uck1ess](https://github.com/5uck1ess), [@dignome](https://github.com/dignome) | [Echo-TTS](echo_tts.md) 44.1 kHz zero-shot voice cloning: 2.8B diffusion transformer in 80-D PCA space, decoded by the Fish S1-DAC autoencoder. Byte-level text, no phonemiser, no reference transcript | | **f5_tts** | TTS, voice cloning | en, ar (Habibi) | Community | [F5-TTS](f5_tts.md) flow-matching DiT — M0 scaffolding, aliases `habibi`/`habibi_tts` | | **glm_tts** | TTS, voice cloning | zh, en | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](glm_tts.md) zero-shot synthesis and voice cloning support | +| **zipvoice** | TTS, voice cloning | zh, en | Community | [ZipVoice](zipvoice.md) k2-fsa TTSZipformer flow-matching zero-shot cloning with Vocos vocoder; distilled 8-step and base variants, hosted GGUF package | | **granite5asr** | ASR | en | Community | [IBM Granite Speech 5.0 470M TurboCTC](granite5asr.md) ultra-fast Conformer-CTC ASR with Shaw relative positional embeddings and ByteLevel BPE | | **inflect_v2** | TTS | en | Community | [Inflect Micro v2 and Nano v2](inflect_v2.md) native FP32 offline synthesis | | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps | diff --git a/docs/community_models/zipvoice.md b/docs/community_models/zipvoice.md new file mode 100644 index 000000000..4e3ec11af --- /dev/null +++ b/docs/community_models/zipvoice.md @@ -0,0 +1,163 @@ +# ZipVoice (community model) + +[ZipVoice](https://github.com/k2-fsa/ZipVoice) (k2-fsa) is a zero-shot voice-cloning TTS built on a +TTSZipformer flow-matching backbone with a Vocos mel-24kHz vocoder. `zipvoice_distill` is the +distilled variant (8 Euler steps, guidance-scale embedding); the base model uses batched +classifier-free guidance. Both share one architecture and one GGUF packaging. + +**Status: inference complete.** Text encoder, duration/ratio conditioning, flow-matching decoder, +and the Vocos vocoder are verified against the reference PyTorch implementation with golden +fixtures (`tests/zipvoice/zipvoice_parity_main.cpp`): per-layer and per-submodule encoder taps, +text conditioning, sampled features, single- and batched velocity fields (odd/even/full lengths, +both timesteps), fbank, and vocos audio — all at cosine 1.0. The Chinese/English text frontend +(`emilia` mode) is verified token-for-token against the upstream `EmiliaTokenizer` +(`tests/zipvoice/zipvoice_zh_tokens_main.cpp`). + +## Highlights + +- Zero-shot voice cloning from a short reference clip + transcript (zh and en) +- Duration prediction from the prompt speaking rate (`speed` option scales it) +- Distill model: 8 Euler steps with guidance-scale embedding; base model: batched CFG +- Chinese + English mixed text via the `emilia` frontend (jieba + pypinyin tables + espeak-ng), + inline pinyin overrides (` `) +- Ready-made GGUF package hosted at [davidxifeng/zipvoice-gguf](https://huggingface.co/davidxifeng/zipvoice-gguf) (Apache-2.0 upstream); local re-conversion with the tools below also works + +## Conversion + +A ready-made, self-contained package is hosted at +[davidxifeng/zipvoice-gguf](https://huggingface.co/davidxifeng/zipvoice-gguf) — the model +manager downloads it directly (`zipvoice-distill-orig.gguf`, flow-matching model + bundled +Vocos + embedded frontend sidecars). To rebuild it locally: + +```bash +# 1. stage the Chinese frontend tables (requires the upstream ZipVoice python env +# for pypinyin; writes zh_chars.tsv / zh_phrases.tsv / zh_syllables.tsv and +# downloads the pinned jieba dictionaries next to tokens.txt) +python3 tools/community_models/export_zipvoice_zh_dict.py \ + --output-dir /models/ZipVoice/zipvoice_distill + +# 2. flatten + package (torch checkpoint -> safetensors -> GGUF with model.* and vocos.*) +python3 tools/community_models/convert_zipvoice.py \ + --model-dir /models/ZipVoice/zipvoice_distill \ + --vocos /models/vocos-mel-24khz/vocos.safetensors \ + --converter build/bin/audiocpp_gguf +``` + +The converter needs `tokens.txt` + `model.json` + `model.pt` in `--model-dir` (the HF +`k2-fsa/ZipVoice` `zipvoice_distill` snapshot layout). When the directory was staged by +`export_zipvoice_zh_dict.py`, the `zh_*` frontend sidecars are embedded into the GGUF alongside +`tokens.txt` / `model.json`, so a converted package is a single self-sufficient file (loose +copies are still staged next to it for the directory layout). With `--safetensors-only` it +stops at the development format (`zipvoice-orig.safetensors` + config + vocab in one +directory) — an intermediate for tooling and the direct synthesis API; the CLI loads GGUF +packages. + +## CLI usage + +```bash +# English cloning (espeak frontend, the default) +audiocpp_cli --task clon --family zipvoice \ + --model /models/ZipVoice-Distill-GGUF/zipvoice-distill-orig.gguf \ + --session-option zipvoice.espeak_library_path=/opt/homebrew/lib/libespeak-ng.dylib \ + --voice-ref prompt.wav --reference-text "Reference transcript." \ + --text "Text to synthesize." --out out.wav + +# Chinese / mixed text (emilia frontend, built in; the zh_* tables are embedded in the +# GGUF, or staged by export_zipvoice_zh_dict.py next to it) +audiocpp_cli --task clon --family zipvoice \ + --model ... \ + --session-option zipvoice.espeak_library_path=/opt/homebrew/lib/libespeak-ng.dylib \ + --voice-ref prompt.wav --reference-text "参考文本。" \ + --text "要合成的文本。" --out out.wav +``` + +The text frontend is fixed to the EmiliaTokenizer pipeline (zh/en/mixed; the upstream +default): `tokenizer` is no longer a session option. Session options: `zipvoice.vocos_path` +(only for safetensors checkpoints without a bundled vocoder), `zipvoice.espeak_library_path`, +`zipvoice.espeak_data_path`, `zipvoice.num_inference_steps`, `zipvoice.guidance_scale`, +`zipvoice.t_shift`. Requests accept `reference_text` (required), `guidance_scale`, +`num_inference_steps`, `t_shift`, `speed`, `feat_scale`, `target_rms`, `seed`, `lang` (espeak +voice for English segments), and `token_ids`/`prompt_token_ids` to bypass the frontend entirely +(direct API callers can also still select the espeak/simple frontends through +`ZipVoiceSynthesisRequest::tokenizer`). + +## Frontend details (emilia mode) + +The upstream default `EmiliaTokenizer` pipeline is reproduced exactly: +Chinese text normalization (framework `ChineseTextNormalizer`) → punctuation mapping → jieba +segmentation (a model-local port of the Jieba maximum-probability DAG and four-state BMES HMM, +`src/community_models/zipvoice/jieba_segmenter.cpp`, same dictionaries as python jieba) → pypinyin +readings from baked tables (TONE3 syllables split into initial+`0` / final+tone tokens) → tone sandhi +(3rd-tone runs, 一/不) applied per jieba word → token mapping with OOV skipping; English runs are +phonemized with espeak-ng. Verification compares C++ token ids against the upstream tokenizer +run in the ZipVoice python environment (`tests/zipvoice/zh_reference_ids.py`). + +The jieba segmentation port is derived from [cppjieba](https://github.com/yanyiwu/cppjieba) +(Copyright (c) 2013 Yanyi Wu) and [jieba](https://github.com/fxsjy/jieba) (Copyright (c) 2012 Sun +Junyi), both MIT licensed; the `zh_jieba_dict.txt` / `zh_hmm_model.txt` resources are the jieba +dictionaries and are embedded in the GGUF. + +## Performance + +The measured ZipVoice-Distill ggml Metal runs achieve **RTF < 1**, the target +for community models. RTF is synthesis wall time divided by generated audio +duration; lower is better. + +Measured on **Apple M3**, **Metal** backend, on **2026-09-18**, with a +Release build and 8 CPU threads, using +`zipvoice-distill-orig.gguf`, 8 Euler steps, guidance scale 3, time shift 0.5, +speed 1, seed 666, and the 24 kHz Vocos vocoder. Each reference pair was tested +with one discarded warmup followed by three timed calls in the same process. +Wall time includes tokenization, reference preprocessing, feature extraction, +text encoding, the solver, and Vocos. Model loading and initial compilation +are excluded by warmup; reference-file reading and output WAV writing are +outside the timer. + +| Reference audio / transcript | Generated audio | Mean synthesis wall time | Mean RTF | Meets RTF < 1 | +|---|---:|---:|---:|---| +| `zh-male-ref.wav` / `zh-male-ref.txt` | 19.115 s | 4.697 s | **0.2457** | Yes | +| `zh-ref.wav` / `zh-ref-text.txt` | 12.843 s | 2.332 s | **0.1816** | Yes | + +The individual measured RTFs were `0.245406`, `0.245910`, `0.245799` for +the male reference and `0.181118`, `0.181592`, `0.182100` for the female +reference. Both used this exact target text, including the space in “宁 静”: + +> 风声渐息，落叶不再沙沙作响。小狐狸缓缓合上双眼，在平缓绵长的呼吸声中，安然步入宁 静的梦境。 + +The measured implementation includes the session-owned runtime, framework Vocos graph, and +model-local Jieba implementation. It measures the direct synthesis API with +one persistent `ZipVoiceComputeDevice` per reference pair; session-level +long-text chunking is not exercised. The two benchmarks ran sequentially. +The discarded first calls took 6.744 s (male) and 2.772 s (female), including +lazy model loading and graph/kernel setup but excluding resource-bundle +resolution and input-file reading. + +Both cases meet the community-model target. Results cover these two inputs +on one machine; CPU and CUDA performance are not established by this measurement. +Local benchmark source, full logs, and output WAVs are saved in +`outputs/zipvoice-metal-current-20260918/` (not distributed with the model). + +## Parity + +Select `--backend metal` for Apple GPU inference or `--backend cpu` for the +CPU reference path. The text encoder, flow decoder and Vocos backbone run on +the selected backend; feature extraction and the final ISTFT run on the host. +The direct C++ API defaults to `BestAvailable`, while sessions honor the requested +backend, including an explicit CPU selection. + +CPU/Metal parity is tested on Apple M3 with the Distill GGUF, including odd +sequence lengths, batched velocity evaluation, Vocos and eight-step synthesis. +ZipVoice requests F32 matrix products to avoid FP16 staging error accumulating +through the sampler. CUDA uses the same graphs but has not been verified on +hardware in this GPU validation run. + +```bash +# golden fixtures: tests/zipvoice/build_reference.py (upstream env) + parity harness +build/bin/zipvoice_parity reference.npz reference.npz vocos.safetensors out.wav +build/bin/zipvoice_zh_tokens zh_reference_ids.json +``` + +Weights and runtime graphs are owned by the `ZipVoiceComputeDevice` runtime (one +per session), so callers injecting a borrowed backend must release the device +runtime (for example with `zipvoice_clear_runtime(device)`) before destroying +that backend, after all ZipVoice calls using it have finished. diff --git a/docs/espeak_phonemizer.md b/docs/espeak_phonemizer.md index a48ab5f30..62094fabd 100644 --- a/docs/espeak_phonemizer.md +++ b/docs/espeak_phonemizer.md @@ -5,7 +5,11 @@ eSpeak-ng. SanoTTS (E2M and Piper frontends) and Inflect v2 use it. Other models including the separate Kokoro preview, can use the same adapter without copying dynamic-library loading or process-global state management. -By default users provide an installed shared library and its matching data. +By default users provide an installed shared library and its matching data. With no explicit +session paths, the adapter first tries common install locations (Homebrew +`/opt/homebrew/lib` and `/usr/local/lib` on macOS, multiarch lib directories on Linux) by +soname, and eSpeak then falls back to its build-time data path; pass +`espeak_library_path` / `espeak_data_path` when the install lives elsewhere. Alternatively, `AUDIOCPP_STATIC_ESPEAK=ON` builds the pinned eSpeak-ng 1.52.0 source and statically links its code into both CLI and server. No eSpeak DLL or `.so` is required in that mode. Existing explicit library/data session options still work. diff --git a/external/ggml/src/ggml-metal/ggml-metal-device.cpp b/external/ggml/src/ggml-metal/ggml-metal-device.cpp index 8f11f92a2..d1f421024 100644 --- a/external/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/external/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -752,12 +752,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_ return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_metal_library_t lib, const ggml_tensor * op) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_metal_library_t lib, const ggml_tensor * op) { char base[256]; char name[256]; const ggml_type tsrc0 = op->src[0]->type; - const ggml_type tsrc1 = op->src[1]->type; + const ggml_type tsrc1 = op->src[1]->type; + const bool full_f32 = tsrc0 == GGML_TYPE_F32 && tsrc1 == GGML_TYPE_F32 && + ggml_get_op_params_i32(op, 0) == GGML_PREC_F32; const bool bc_inp = op->src[0]->ne[0] % 32 != 0; @@ -776,7 +778,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_meta const int16_t r2 = (int16_t) (ne12 / op->src[0]->ne[2]); const int16_t r3 = (int16_t) (ne13 / op->src[0]->ne[3]); - snprintf(base, 256, "kernel_mul_mm_%s_%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1)); + snprintf(base, 256, "kernel_mul_mm_%s_%s%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1), + full_f32 ? "_prec_f32" : ""); snprintf(name, 256, "%s_bci=%d_bco=%d_ne12=%d_ne13=%d_r2=%d_r3=%d", base, bc_inp, bc_out, ne12, ne13, r2, r3); @@ -800,13 +803,13 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_meta res.nr0 = NRA; res.nr1 = NRB; - const size_t smem_a = NRA * N_MM_NK_TOTAL * sizeof(ggml_fp16_t); + const size_t smem_a = NRA * N_MM_NK_TOTAL * (full_f32 ? sizeof(float) : sizeof(ggml_fp16_t)); res.smem = smem_a; } else { res.nr0 = 64; res.nr1 = 32; - res.smem = bc_out ? 8192 : (4096 + 2048); + res.smem = full_f32 ? (8192 + 4096) : (bc_out ? 8192 : (4096 + 2048)); } res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y; diff --git a/external/ggml/src/ggml-metal/ggml-metal-ops.cpp b/external/ggml/src/ggml-metal/ggml-metal-ops.cpp index 40a5dca40..ab33d920d 100644 --- a/external/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/external/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2371,7 +2371,15 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { !ggml_is_transposed(op->src[1]) && // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel - props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) { + // Short F32 contractions (e.g. 32-dim QK attention) still benefit + // from tiled MM when both output axes are large enough. Keep the + // existing MV choice for small outputs and other precision modes. + props_dev->has_simdgroup_mm && + (ne00 >= 64 || (ne00 >= 32 && ne01 >= 64 && ne11 >= 32 && + op->src[0]->type == GGML_TYPE_F32 && + op->src[1]->type == GGML_TYPE_F32 && + ggml_get_op_params_i32(op, 0) == GGML_PREC_F32)) && + ne11 > ne11_mm_min) { //GGML_LOG_INFO("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); // some Metal matrix data types require aligned pointers diff --git a/external/ggml/src/ggml-metal/ggml-metal.metal b/external/ggml/src/ggml-metal/ggml-metal.metal index b71b83c68..4c0f7ad39 100644 --- a/external/ggml/src/ggml-metal/ggml-metal.metal +++ b/external/ggml/src/ggml-metal/ggml-metal.metal @@ -7876,8 +7876,15 @@ kernel void kernel_cpy_t_t( ushort3 ntg[[threads_per_threadgroup]]) { const int i03 = tgpig[2]; const int i02 = tgpig[1]; - const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0]; - const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; + const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0]; + const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; + + // The last threadgroup can contain padded rows (including ne01 == 1 + // after a permutation). Those threads must not read or overwrite the + // next channel/batch through its different source strides. + if (i01 >= args.ne01) { + return; + } const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; @@ -10224,7 +10231,7 @@ kernel void kernel_mul_mm( ushort sgitg[[simdgroup_index_in_threadgroup]]) { threadgroup S0 * sa = (threadgroup S0 *)(shmem); - threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096); + threadgroup S1 * sb = (threadgroup S1 *)(shmem + 64*32*sizeof(S0)); constexpr int NR0 = 64; constexpr int NR1 = 32; @@ -10871,7 +10878,9 @@ template [[host_name("kernel_set_rows_iq4_nl_i32")]] kernel set_rows_q32_t kerne typedef decltype(kernel_mul_mm) mul_mm_t; -template [[host_name("kernel_mul_mm_f32_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_f32_f32")]] kernel mul_mm_t kernel_mul_mm; +// GGML_PREC_F32 must preserve F32 operands instead of staging them as half. +template [[host_name("kernel_mul_mm_f32_f32_prec_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; diff --git a/include/engine/community_models/zipvoice/emilia_tokenizer.h b/include/engine/community_models/zipvoice/emilia_tokenizer.h new file mode 100644 index 000000000..f4def0006 --- /dev/null +++ b/include/engine/community_models/zipvoice/emilia_tokenizer.h @@ -0,0 +1,80 @@ +#pragma once + +// Chinese/English mixed text frontend for ZipVoice, mirroring the upstream +// EmiliaTokenizer (zipvoice/tokenizer/tokenizer.py). Chinese segments are +// converted with baked pypinyin tables (tone3 syllables split into +// initial+"0" / final+tone tokens, e.g. "w0 o3") plus run-time tone sandhi; +// English segments are phonemized with eSpeak-ng; overrides use +// the bare-syllable table. Output token ids follow tokens.txt with OOV +// tokens skipped, exactly like the reference. + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::audio { +class EspeakPhonemizer; +} + +namespace engine::models::zipvoice { + +class JiebaSegmenter; + +class EmiliaTokenizer { +public: + struct EspeakConfig { + std::string library_path; // optional: libespeak-ng path + std::string data_path; // optional: espeak-ng data directory + std::string lang = "en-us"; + }; + + // Frontend table locations. The spec-registered resource bundle hands + // over materialized GGUF-embedded sidecars; development directories and + // direct API callers fill the paths from the model directory. + struct TablePaths { + std::filesystem::path chars; // zh_chars.tsv + std::filesystem::path phrases; // zh_phrases.tsv + std::filesystem::path syllables; // zh_syllables.tsv + std::filesystem::path jieba_dict; // zh_jieba_dict.txt + std::filesystem::path hmm_model; // zh_hmm_model.txt + + // Loose-file layout: everything in one model directory. + static TablePaths from_model_dir(const std::filesystem::path & dir) { + return {dir / "zh_chars.tsv", dir / "zh_phrases.tsv", + dir / "zh_syllables.tsv", dir / "zh_jieba_dict.txt", + dir / "zh_hmm_model.txt"}; + } + }; + + // `vocab` is the tokens.txt token -> id map; OOV phones are skipped. + // The Chinese tables are optional only if no Chinese text is encoded. + EmiliaTokenizer(const TablePaths & tables, + const std::unordered_map & vocab, + const EspeakConfig & espeak); + ~EmiliaTokenizer(); // out-of-line: owns an incomplete-type unique_ptr + + std::vector encode(const std::string & text) const; + +private: + std::unordered_map vocab_; + TablePaths tables_; + std::unordered_map> chars_; + std::unordered_map> phrases_; + std::unordered_map> syllables_; + size_t max_phrase_codepoints_ = 0; + EspeakConfig espeak_; + // Lazily created so a pure-Chinese workload never touches eSpeak. + mutable std::unique_ptr phonemizer_; + // Lazily created jieba segmenter (model-local MixSegment port = python + // jieba.cut with HMM): word boundaries define reading lookup and the + // scope of tone-sandhi application, mirroring the reference pipeline + // lazy_pinyin(jieba.cut(text), tone_sandhi=True). + mutable std::unique_ptr segmenter_; + mutable std::mutex segmenter_mutex_; +}; + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/session.h b/include/engine/community_models/zipvoice/session.h new file mode 100644 index 000000000..9e46b733e --- /dev/null +++ b/include/engine/community_models/zipvoice/session.h @@ -0,0 +1,12 @@ +#pragma once + +#include "engine/framework/runtime/model.h" + +#include +#include + +namespace engine::models::zipvoice { + +std::shared_ptr make_zipvoice_loader(); + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/synthesize.h b/include/engine/community_models/zipvoice/synthesize.h new file mode 100644 index 000000000..869f82766 --- /dev/null +++ b/include/engine/community_models/zipvoice/synthesize.h @@ -0,0 +1,151 @@ +#pragma once + +#include "engine/framework/core/backend.h" + +#include +#include +#include +#include + +namespace engine::assets { +class ResourceBundle; +} + +namespace engine::models::zipvoice { + +class ZipVoiceRuntimeState; +std::shared_ptr make_zipvoice_runtime_state(); + +// Compute device for the whole zipvoice stack (text encoder, flow-matching +// decoder, Vocos backbone). GPU first: the default BestAvailable resolves to +// Metal/CUDA when a usable accelerator exists and falls back to CPU +// otherwise; `threads` applies to CPU compute. A runtime that already owns a +// backend may inject it via `backend`. Destroy/reset this device runtime before +// freeing a borrowed backend. Copies share runtime ownership and serialize use. +struct ZipVoiceComputeDevice { + core::BackendType backend_type = core::BackendType::BestAvailable; + int device_index = 0; + int threads = 0; // 0 = hardware concurrency + ggml_backend_t backend = nullptr; + std::shared_ptr runtime = make_zipvoice_runtime_state(); +}; + +// Release this session/device's weights and graphs; no process-global cache. +void zipvoice_clear_runtime(const ZipVoiceComputeDevice & device); + +struct ZipVoiceSynthesisRequest { + std::string text; + std::string ref_text; + std::vector ref_audio; + int ref_sample_rate = 24000; + int ref_channels = 1; // ref_audio is interleaved + std::vector token_ids; // optional: pre-tokenized target text + std::vector prompt_token_ids; // optional: pre-tokenized ref text + std::string tokenizer = "emilia"; // fixed frontend: zh/en/mixed (espeak for en runs) + std::string lang = "en-us"; + std::string espeak_library_path; + std::string espeak_data_path; + int num_steps = 8; + float guidance_scale = 3.0F; + float t_shift = 0.5F; + float speed = 1.0F; + float feat_scale = 0.1F; + float target_rms = 0.1F; + uint32_t seed = 666; + bool fixed_seed = true; +}; + +struct ZipVoiceSynthesisResult { + std::vector audio; // 24 kHz mono + int sample_rate = 24000; + double model_seconds = 0.0; // wall time of the flow + vocoder + double audio_seconds = 0.0; // generated audio duration +}; + +// Full pipeline: tokenize (if needed) -> fbank -> duration prediction -> +// Euler flow-matching sampling -> Vocos decode. `model_path` is the GGUF +// package or the safetensors development directory. `resources` (optional) +// is the spec-resolved bundle: registered sidecars (tokens, model_config, +// zh_* tables, materialized from an embedded-sidecar GGUF or found in the +// development directory) take priority over loose files next to the +// checkpoint; direct API callers (parity harnesses) may pass nullptr. +ZipVoiceSynthesisResult zipvoice_synthesize( + const std::string & model_path, + const std::string & vocos_path, + const ZipVoiceSynthesisRequest & request, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// --- test hooks (parity harness) ------------------------------------------- + +// Backend Vocos path using the same cached model/backend as synthesis. +std::vector zipvoice_vocos_decode_on_device( + const std::string & model_path, + const std::string & vocos_path, + const std::vector & mel_rows, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources = nullptr); + +// Text-condition stage: token ids -> text_condition [T, feat_dim] given +// prompt feature length. Mirrors forward_text_inference_ratio_duration. +std::vector zipvoice_text_condition( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + int64_t prompt_features_len, + float speed, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// Raw text-encoder output for pre-tokenized ids (pad token appended +// internally, mirroring pad_labels): returns [S+1, feat_dim] row-major. +// `layer_taps` (optional) receives per-encoder-layer outputs as [S+1, C] +// rows for parity bisecting. +struct ZipVoiceLayerTaps { + std::vector> layers; // per-layer outputs [S+1, C] + std::vector> stages; // first-layer submodule taps +}; +std::vector zipvoice_text_encoder_raw( + const std::string & model_path, + const std::vector & token_ids, + ZipVoiceLayerTaps * layer_taps = nullptr, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// One flow-matching velocity evaluation at time t (single batch, no CFG). +std::vector zipvoice_velocity( + const std::string & model_path, + const std::vector & xt, // [T, feat_dim] + const std::vector & text_condition, + const std::vector & speech_condition, + int64_t features_len, + float t, + float guidance_scale, + const ZipVoiceComputeDevice & device = {}, + int batch_size = 1, + const engine::assets::ResourceBundle * resources = nullptr); + +// Full sampler: x0 -> x1 (mirrors ZipVoice::sample for duration="predict"). +std::vector zipvoice_sample( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + const std::vector & prompt_features, // [T_prompt, feat_dim], scaled + int64_t prompt_features_len, + const std::vector & x0, + int num_steps, + float guidance_scale, + float t_shift, + float speed, + const ZipVoiceComputeDevice & device = {}, + const engine::assets::ResourceBundle * resources = nullptr); + +// Log-mel filterbank identical to VocosFbank (24 kHz, 100 mels, hop 256, +// power=1, log clamp 1e-7, htk scale), with lhotse frame-count alignment. +std::vector zipvoice_logmel(const std::vector & wav); + +// Vocos mel-24kHz decode (same weights as f5_tts uses). +std::vector zipvoice_vocos_decode( + const std::string & vocos_path, const std::vector & mel_frames); + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/weights.h b/include/engine/community_models/zipvoice/weights.h new file mode 100644 index 000000000..dc56a5730 --- /dev/null +++ b/include/engine/community_models/zipvoice/weights.h @@ -0,0 +1,111 @@ +#pragma once + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/module.h" + +#include +#include +#include +#include + +namespace engine::models::zipvoice { + +// Architecture hyper-parameters, mirroring model.json "model" + tokenizer info. +struct ZipVoiceConfig { + std::vector fm_downsampling_factor = {1, 2, 4, 2, 1}; + std::vector fm_num_layers = {2, 2, 4, 4, 4}; + std::vector fm_cnn_kernel = {31, 15, 7, 15, 31}; + int fm_feedforward_dim = 1536; + int fm_num_heads = 4; + int fm_dim = 512; + int text_num_layers = 4; + int text_feedforward_dim = 512; + int text_cnn_kernel = 9; + int text_num_heads = 4; + int text_dim = 192; + int time_embed_dim = 192; + int text_embed_dim = 192; + int query_head_dim = 32; + int value_head_dim = 12; + int pos_head_dim = 4; + int pos_dim = 48; + int feat_dim = 100; + int vocab_size = 360; // from tokens.txt + int pad_id = 0; + int sampling_rate = 24000; + bool guidance_scale_embed = false; // ZipVoice-Distill +}; + +// One Zipformer2 encoder layer's weights (torch module names). +struct ZipLayerWeights { + core::TensorValue bypass_scale; // [C] + core::TensorValue bypass_mid_scale; // [C] + core::TensorValue attn_in_proj_w; // [C, (2*qh+ph)*H] + core::TensorValue attn_in_proj_b; + core::TensorValue linear_pos_w; // [H*ph, pos_dim] + core::TensorValue sa1_in_w, sa1_in_b; // [H*vh, C] + core::TensorValue sa1_out_w, sa1_out_b; // [C, H*vh] + core::TensorValue sa2_in_w, sa2_in_b; + core::TensorValue sa2_out_w, sa2_out_b; + core::TensorValue ff1_in_w, ff1_in_b; + core::TensorValue ff1_out_w, ff1_out_b; + core::TensorValue ff2_in_w, ff2_in_b; + core::TensorValue ff2_out_w, ff2_out_b; + core::TensorValue ff3_in_w, ff3_in_b; + core::TensorValue ff3_out_w, ff3_out_b; + core::TensorValue na_in_w, na_in_b; // [3*hidden, C] + core::TensorValue na_out_w, na_out_b; // [C, hidden] + core::TensorValue cm1_in_w, cm1_in_b; + core::TensorValue cm1_conv_w, cm1_conv_b; // [C, 1, k], [C] + core::TensorValue cm1_out_w, cm1_out_b; + core::TensorValue cm2_in_w, cm2_in_b; + core::TensorValue cm2_conv_w, cm2_conv_b; + core::TensorValue cm2_out_w, cm2_out_b; + core::TensorValue norm_bias; // [C] + float norm_log_scale = 0.0F; +}; + +struct ZipStackWeights { + std::vector layers; + // present when the stack has a time embedding projection + core::TensorValue time_proj_w, time_proj_b; // [C, time_dim] + // present when downsampling_factor > 1 + std::vector downsample_bias; // [ds] (softmax applied at load) + core::TensorValue out_combiner_scale; // [C] +}; + +struct TTSZipformerWeights { + core::TensorValue in_proj_w, in_proj_b; + core::TensorValue out_proj_w, out_proj_b; + core::TensorValue embed_w; // only for text encoder: [vocab, text_embed_dim] + // fm decoder only: + core::TensorValue time_mlp0_w, time_mlp0_b; // [2*tdim, tdim] + core::TensorValue time_mlp2_w, time_mlp2_b; // [tdim, 2*tdim] + core::TensorValue guidance_embed_w; // [tdim, tdim] no bias (distill) + std::vector stacks; +}; + +struct ZipVoiceWeights { + std::shared_ptr store; + TTSZipformerWeights fm_decoder; + TTSZipformerWeights text_encoder; +}; + +// Loads a converted checkpoint. `prefix` is "model" for GGUF packages and "" +// for the development safetensors export. Throws on shape mismatch. +// `config_path` (optional) overrides `model_dir / "model.json"` for +// spec-registered (materialized) sidecars. +ZipVoiceConfig load_zipvoice_config( + const std::filesystem::path & model_dir, + const class engine::assets::TensorSource * probe, + const std::filesystem::path * config_path = nullptr); + +ZipVoiceWeights load_zipvoice_weights( + const engine::assets::TensorSource & source, + const std::string & prefix, + const ZipVoiceConfig & config, + ggml_backend_t backend, + core::BackendType backend_type); + +} // namespace engine::models::zipvoice diff --git a/include/engine/community_models/zipvoice/zipformer.h b/include/engine/community_models/zipvoice/zipformer.h new file mode 100644 index 000000000..239e70111 --- /dev/null +++ b/include/engine/community_models/zipvoice/zipformer.h @@ -0,0 +1,84 @@ +#pragma once + +#include "engine/community_models/zipvoice/weights.h" + +#include +#include +#include + +#include +#include +#include + +namespace engine::models::zipvoice { + +// Raw-ggml graph builders for the two ZipVoice networks. +// +// Physical layouts (ne0 = fastest): +// activations [C, T, B]; attention weights [src, tgt, H, B] +// All softmaxes run over ne0 (= keys). +// +// Memory model: leaves + constants live in a dedicated tensor context with +// their own backend buffer (ggml_backend_alloc_ctx_tensors) so the graph +// arena never aliases them; per-call values are uploaded into that buffer. + +struct ZipVoiceGraphResources { + ggml_context * ctx = nullptr; + ggml_context * tensor_ctx = nullptr; + ggml_backend_buffer_t buffer = nullptr; + ggml_cgraph * graph = nullptr; + ggml_backend_t backend = nullptr; + void * gallocr = nullptr; + ZipVoiceGraphResources() = default; + ~ZipVoiceGraphResources(); + ZipVoiceGraphResources(const ZipVoiceGraphResources &) = delete; + ZipVoiceGraphResources & operator=(const ZipVoiceGraphResources &) = delete; + ZipVoiceGraphResources(ZipVoiceGraphResources && other) noexcept + : ctx(std::exchange(other.ctx, nullptr)), tensor_ctx(std::exchange(other.tensor_ctx, nullptr)), + buffer(std::exchange(other.buffer, nullptr)), graph(std::exchange(other.graph, nullptr)), + backend(std::exchange(other.backend, nullptr)), gallocr(std::exchange(other.gallocr, nullptr)) {} +}; + +struct FmDecoderGraph : ZipVoiceGraphResources { + ggml_tensor * x_cat = nullptr; // leaf [3F, T, B] + ggml_tensor * time_emb = nullptr; // leaf [time_embed_dim] + ggml_tensor * guidance_emb = nullptr; // leaf [time_embed_dim] (distill) + ggml_tensor * pad_bias[8] = {}; // leaves [T_s, 1, 1, 1] (0/-1000) + ggml_tensor * conv_gate[8] = {}; // leaves [1, T_s, 1] (0/1) + ggml_tensor * output = nullptr; // [F, T, B] + int64_t T = 0; + int64_t B = 1; + bool cuda = false; +}; + +struct TextEncoderGraph : ZipVoiceGraphResources { + ggml_tensor * token_ids = nullptr; // leaf [S] i32 + ggml_tensor * pad_bias[1] = {}; // [S, 1, 1, 1] + ggml_tensor * conv_gate[1] = {}; // [1, S, 1] + ggml_tensor * output = nullptr; // [feat_dim, S] + std::vector layer_taps; // per-layer outputs [C, S] + std::vector stage_taps; // first-layer submodule taps + int64_t S = 0; + bool cuda = false; +}; + +// Builds the flow-matching decoder graph. `with_guidance` wires the distill +// guidance-scale embedding input (ignored for the base model). +FmDecoderGraph build_fm_decoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t T, + int64_t B, + bool with_guidance, + bool cuda_backend, + ggml_backend_t backend); + +// Builds the text encoder graph (single stack, no time embedding). +TextEncoderGraph build_text_encoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t S, + bool cuda_backend, + ggml_backend_t backend); + +} // namespace engine::models::zipvoice diff --git a/include/engine/framework/modules/convnext_modules.h b/include/engine/framework/modules/convnext_modules.h new file mode 100644 index 000000000..5244a12a1 --- /dev/null +++ b/include/engine/framework/modules/convnext_modules.h @@ -0,0 +1,18 @@ +#pragma once +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" + +namespace engine::modules { +// Vocos/F5-style non-causal ConvNeXt, logical [batch, frames, channels]. +struct ConvNeXt1dWeights { + DepthwiseConv1dWeights depthwise; + NormWeights norm; + LinearWeights expansion; + LinearWeights projection; + core::TensorValue gamma; +}; +core::TensorValue build_convnext1d( + core::ModuleBuildContext & ctx, const core::TensorValue & input, + const ConvNeXt1dWeights & weights); +} diff --git a/include/engine/framework/modules/vocoders/vocos_vocoder.h b/include/engine/framework/modules/vocoders/vocos_vocoder.h new file mode 100644 index 000000000..c6fa94231 --- /dev/null +++ b/include/engine/framework/modules/vocoders/vocos_vocoder.h @@ -0,0 +1,34 @@ +#pragma once +#include "engine/framework/core/backend.h" +#include "engine/framework/modules/convnext_modules.h" +#include +#include +#include + +namespace engine::modules { +struct VocosBackboneWeights { + Conv1dWeights embed; + NormWeights input_norm, final_norm; + LinearWeights head; + std::vector blocks; +}; +// Shared F5/ZipVoice backbone, logical [batch, frames, mel] -> spectral rows. +core::TensorValue build_vocos_backbone(core::ModuleBuildContext & ctx, + const core::TensorValue & mel, const VocosBackboneWeights & weights); + +// Generalized F5/Vocos mel-24khz graph: Conv1d, ConvNeXt, LN, spectral head. +// Owns backend weights and one runtime graph/ISTFT workspace. The backend is +// borrowed and must outlive this object. Callers serialize decode calls. +class VocosVocoder { +public: + VocosVocoder(const std::string & checkpoint, ggml_backend_t backend, int threads); + ~VocosVocoder(); + VocosVocoder(const VocosVocoder &) = delete; + VocosVocoder & operator=(const VocosVocoder &) = delete; + std::vector decode(const std::vector & mel_rows); + size_t cached_graph_count() const; +private: + class Impl; + std::unique_ptr impl_; +}; +} diff --git a/model_specs/zipvoice.json b/model_specs/zipvoice.json new file mode 100644 index 000000000..4d71feca4 --- /dev/null +++ b/model_specs/zipvoice.json @@ -0,0 +1,221 @@ +{ + "schema_version": 1, + "family": "zipvoice", + "display_name": "ZipVoice", + "description": "Community ZipVoice / ZipVoice-Distill flow-matching TTS with a TTSZipformer backbone (U-Net downsampling stacks, compact relative position attention), duration prediction from prompt ratio, Euler solver with t-shift and classifier-free guidance, and Vocos mel-24kHz vocoder (k2-fsa/ZipVoice).", + "category": "tts", + "status": "community", + "tasks": [ + "tts", + "clone" + ], + "modes": [ + "offline" + ], + "languages": [ + "en", + "zh" + ], + "runtime": { + "tags": [ + "server" + ] + }, + "capabilities": { + "clone": [ + "speaker_reference" + ] + }, + "options": { + "request": [ + { + "name": "reference_text", + "type": "string", + "description": "Transcript matching the reference voice audio; required for zero-shot cloning.", + "required": true + }, + { + "name": "guidance_scale", + "type": "float", + "description": "Classifier-free guidance scale; default 3.0 for ZipVoice-Distill (guidance-scale embedding), 1.0 for ZipVoice (batched CFG). 0 disables guidance.", + "required": false, + "min": 0.0, + "max": 10.0, + "default": 3.0 + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Euler ODE steps; default 8 for ZipVoice-Distill, 16 for ZipVoice.", + "required": false, + "min": 1, + "max": 64, + "default": 8 + }, + { + "name": "t_shift", + "type": "float", + "description": "Shift timesteps toward low SNR (smaller = stronger shift); default 0.5.", + "required": false, + "min": 0.05, + "max": 1.0, + "default": 0.5 + }, + { + "name": "speed", + "type": "float", + "description": "Speech speed multiplier applied through the prompt-duration ratio; default 1.0.", + "required": false, + "min": 0.5, + "max": 2.0, + "default": 1.0 + }, + { + "name": "feat_scale", + "type": "float", + "description": "Feature scale applied to log-mel features; default 0.1 (reference default).", + "required": false, + "min": 0.01, + "max": 1.0, + "default": 0.1 + }, + { + "name": "target_rms", + "type": "float", + "description": "Target RMS for prompt loudness normalization; 0 disables. Default 0.1.", + "required": false, + "min": 0.0, + "max": 1.0, + "default": 0.1 + }, + { + "name": "seed", + "type": "int", + "description": "Noise seed; default 666 (reference default).", + "required": false, + "min": 0, + "default": 666 + }, + { + "name": "lang", + "type": "string", + "description": "espeak language for phonemization; default en-us.", + "required": false, + "default": "en-us" + } + ], + "session": [ + { + "name": "vocos_path", + "type": "string", + "description": "Path to the Vocos vocoder checkpoint (vocos.safetensors or GGUF); required unless bundled in the model GGUF or placed next to the checkpoint.", + "required": false + }, + { + "name": "guidance_scale", + "type": "float", + "description": "Default guidance scale for requests that do not set it.", + "required": false, + "min": 0.0, + "max": 10.0, + "default": 3.0 + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Default Euler steps for requests that do not set it.", + "required": false, + "min": 1, + "max": 64, + "default": 8 + }, + { + "name": "t_shift", + "type": "float", + "description": "Default timestep shift for requests that do not set it.", + "required": false, + "min": 0.05, + "max": 1.0, + "default": 0.5 + }, + { + "name": "espeak_library_path", + "type": "string", + "description": "Path to the espeak-ng shared library (e.g. /opt/homebrew/lib/libespeak-ng.dylib) for tokenizer=espeak.", + "required": false + }, + { + "name": "espeak_data_path", + "type": "string", + "description": "Path to the espeak-ng data directory or package for tokenizer=espeak.", + "required": false + } + ], + "load": [] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "davidxifeng/zipvoice-gguf", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "zipvoice_distill_gguf", + "display_name": "ZipVoice-Distill GGUF (local conversion)", + "description": "Self-contained GGUF: flow-matching model, bundled Vocos vocoder and the embedded text-frontend sidecars (tokens, config, zh tables) in one file. Converted from k2-fsa/ZipVoice with export_zipvoice_zh_dict.py + convert_zipvoice.py.", + "default": true, + "format": "gguf", + "precision": "orig", + "target_directory": "ZipVoice-Distill-GGUF", + "files": [ + "zipvoice-distill-orig.gguf" + ] + } + ], + "dependencies": [], + "ui": { + "recommended_package": "zipvoice_distill_gguf", + "tags": [ + "TTS", + "Clone" + ], + "docs": [ + "docs/community_models/zipvoice.md" + ] + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "tokens": "model:tokens.txt", + "model_config": "model:model.json" + }, + "optional_files": { + "zh_chars": "model:zh_chars.tsv", + "zh_phrases": "model:zh_phrases.tsv", + "zh_syllables": "model:zh_syllables.tsv", + "zh_jieba_dict": "model:zh_jieba_dict.txt", + "zh_hmm_model": "model:zh_hmm_model.txt" + }, + "tensors": { + "model": { + "source": "weights:", + "prefix": "model" + } + }, + "optional_tensors": { + "vocos_vocoder": { + "source": "weights:", + "prefix": "vocos" + } + } + } + ] +} diff --git a/src/community_models/zipvoice/emilia_tokenizer.cpp b/src/community_models/zipvoice/emilia_tokenizer.cpp new file mode 100644 index 000000000..b9bcc27c5 --- /dev/null +++ b/src/community_models/zipvoice/emilia_tokenizer.cpp @@ -0,0 +1,342 @@ +#include "engine/community_models/zipvoice/emilia_tokenizer.h" + +#include "jieba_segmenter.h" + +#include "engine/framework/audio/espeak_phonemizer.h" +#include "engine/framework/text/chinese_normalization.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +// --------------------------------------------------------------------------- +// UTF-8 helpers + +std::vector split_utf8(const std::string & text) { + std::vector out; + for (size_t i = 0; i < text.size();) { + const auto lead = static_cast(text[i]); + size_t width = 1; + if (lead >= 0xF0) width = 4; + else if (lead >= 0xE0) width = 3; + else if (lead >= 0xC0) width = 2; + out.emplace_back(text, i, std::min(width, text.size() - i)); + i += out.back().size(); + } + return out; +} + +uint32_t utf8_to_codepoint(const std::string & ch) { + const auto lead = static_cast(ch[0]); + if (lead < 0x80) return lead; + if ((lead & 0xE0) == 0xC0) return (uint32_t(lead & 0x1F) << 6) | + (uint32_t(static_cast(ch[1])) & 0x3F); + if ((lead & 0xF0) == 0xE0) return (uint32_t(lead & 0x0F) << 12) | + (uint32_t(static_cast(ch[1])) & 0x3F) << 6 | + (uint32_t(static_cast(ch[2])) & 0x3F); + return (uint32_t(lead & 0x07) << 18) | + (uint32_t(static_cast(ch[1])) & 0x3F) << 12 | + (uint32_t(static_cast(ch[2])) & 0x3F) << 6 | + (uint32_t(static_cast(ch[3])) & 0x3F); +} + +bool is_han_char(uint32_t cp) { + // Reference EmiliaTokenizer::is_chinese checks U+4E00..U+9FA5 only. + return cp >= 0x4E00 && cp <= 0x9FA5; +} + +bool is_alpha_char(uint32_t cp) { + return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z'); +} + +// --------------------------------------------------------------------------- +// Table loading + +std::unordered_map> load_token_table( + const std::filesystem::path & path) { + std::unordered_map> out; + std::ifstream in(path); + if (!in) throw std::runtime_error("zipvoice: missing Chinese table " + path.string()); + std::string line; + while (std::getline(in, line)) { + if (line.empty()) continue; + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + std::vector tokens; + std::istringstream rest(line.substr(tab + 1)); + std::string token; + while (rest >> token) tokens.push_back(token); + out.emplace(line.substr(0, tab), std::move(tokens)); + } + return out; +} + +// --------------------------------------------------------------------------- +// Text shaping (mirrors EmiliaTokenizer::map_punctuations) + +std::string map_punctuations(std::string text) { + static const std::pair kMap[] = { + {"，", ","}, {"。", "."}, {"！", "!"}, {"？", "?"}, {"；", ";"}, + {"：", ":"}, {"、", ","}, {"‘", "'"}, {"“", "\""}, {"”", "\""}, + {"’", "'"}, {"⋯", "…"}, {"···", "…"}, {"・・・", "…"}, {"...", "…"}, + }; + for (const auto & [from, to] : kMap) { + for (size_t pos = 0; (pos = text.find(from, pos)) != std::string::npos;) { + text.replace(pos, from.size(), to); + pos += to.size(); + } + } + return text; +} + +// --------------------------------------------------------------------------- +// Tone sandhi (pypinyin contrib/tone_sandhi.py semantics) +// +// Scope note: the reference only reaches this with a word the pypinyin +// phrases_dict matched (whole-word scope) or a single unmatched character; +// the caller mirrors that by applying it per phrase-match / per character. + +void apply_tone_sandhi(std::vector & tokens, const std::string & han) { + // Syllables are the tokens whose last char is a tone digit. + std::vector finals; + for (size_t i = 0; i < tokens.size(); ++i) { + const auto & t = tokens[i]; + if (!t.empty() && t.back() >= '1' && t.back() <= '5') finals.push_back(i); + } + const size_t n = finals.size(); + if (n == 0) return; + auto tone = [&](size_t s) { return tokens[finals[s]].back(); }; + auto set_tone = [&](size_t s, char d) { tokens[finals[s]].back() = d; }; + + // Third tone: pypinyin counts the TRAILING consecutive run of '3' + // syllables. A run of exactly 2 turns the first '3' in the word into + // '2'; a longer run turns every '3' but the last into '2'. + size_t run = 0; + for (size_t s = n; s > 0; --s) { + if (tone(s - 1) != '3') break; + ++run; + } + if (run == 2) { + for (size_t s = 0; s < n; ++s) { + if (tone(s) == '3') { set_tone(s, '2'); break; } + } + } else if (run > 2) { + size_t seen = 0; + for (size_t s = 0; s < n; ++s) { + if (tone(s) != '3') continue; + if (++seen == run) break; + set_tone(s, '2'); + } + } + + // 不 / 一 (pypinyin's replace('4','2') only fires when the current tone + // is '4'; the non-4th branch FORCES '4'; a final 一 forces '1'). + const auto han_chars = split_utf8(han); + for (size_t s = 0; s < n && s < han_chars.size(); ++s) { + const bool is_bu = han_chars[s] == "不"; + const bool is_yi = han_chars[s] == "一"; + if (!is_bu && !is_yi) continue; + if (s + 1 < n) { + if (tone(s + 1) == '4') { + if (tone(s) == '4') set_tone(s, '2'); + } else { + set_tone(s, '4'); + } + } else { + set_tone(s, is_yi ? '1' : '4'); + } + } +} + +} // namespace + +// --------------------------------------------------------------------------- +// EmiliaTokenizer + +EmiliaTokenizer::EmiliaTokenizer( + const TablePaths & tables, + const std::unordered_map & vocab, + const EspeakConfig & espeak) + : vocab_(vocab), tables_(tables), espeak_(espeak) { + chars_ = load_token_table(tables_.chars); + phrases_ = load_token_table(tables_.phrases); + syllables_ = load_token_table(tables_.syllables); +} + +EmiliaTokenizer::~EmiliaTokenizer() = default; + +std::vector EmiliaTokenizer::encode(const std::string & raw_text) const { + // phone string -> id + auto emit = [&](const std::vector & phones, std::vector & ids) { + for (const auto & phone : phones) { + if (const auto it = vocab_.find(phone); it != vocab_.end()) { + ids.push_back(it->second); + } + } + }; + + // Segment into (text, lang) runs: zh (Han), en (ASCII letters), other. + // "other" attaches to the current language; a leading "other" run takes + // the language of the following text. + struct Segment { std::string text; char lang; }; + std::vector segments; + { + const auto chars = split_utf8(raw_text); + char lang = 'o'; + std::string current; + for (const auto & ch : chars) { + const auto cp = utf8_to_codepoint(ch); + const char type = is_han_char(cp) ? 'z' : (is_alpha_char(cp) ? 'e' : 'o'); + if (current.empty()) { + current = ch; + lang = type; + } else if (lang == 'o') { + current += ch; + lang = type; + } else if (type == lang || type == 'o') { + current += ch; + } else { + segments.push_back({current, lang}); + current = ch; + lang = type; + } + } + if (!current.empty()) segments.push_back({current, lang}); + } + + // Split / [tag] parts out of every segment. + struct Part { std::string text; char lang; bool bracket; }; + std::vector parts; + for (auto & seg : segments) { + std::string pending; + for (size_t i = 0; i < seg.text.size();) { + const char c = seg.text[i]; + if (c == '<' || c == '[') { + const char close = c == '<' ? '>' : ']'; + const auto end = seg.text.find(close, i + 1); + if (end != std::string::npos) { + if (!pending.empty()) { + parts.push_back({pending, seg.lang, false}); + pending.clear(); + } + parts.push_back({seg.text.substr(i, end - i + 1), seg.lang, true}); + i = end + 1; + continue; + } + } + pending += c; + ++i; + } + if (!pending.empty()) parts.push_back({pending, seg.lang, false}); + } + + std::vector ids; + for (const auto & part : parts) { + if (part.bracket) { + if (part.text.front() == '<' && part.text.back() == '>') { + // Inline pinyin override, e.g. -> "w0 o3". + const std::string syllable = part.text.substr(1, part.text.size() - 2); + if (!syllable.empty() && std::isalpha(static_cast(syllable.front())) && + syllable.back() >= '1' && syllable.back() <= '5') { + if (const auto it = syllables_.find(syllable); it != syllables_.end()) { + emit(it->second, ids); + } + } + } else { + // [tag]: the whole bracketed string is one phone (usually OOV). + emit({part.text}, ids); + } + continue; + } + if (part.lang == 'e') { + // English segment: eSpeak-ng phonemization, one phone per + // codepoint (matches the audio.cpp espeak frontend convention). + // eSpeak treats sentence punctuation as clause markers and drops + // it from the phoneme stream, but the reference keeps it as + // tokens, so re-append trailing punctuation here. + static const std::string kTrailingPunct = ".,!?;:"; + std::string text = part.text; + std::string trailing; + while (!text.empty() && kTrailingPunct.find(text.back()) != std::string::npos) { + trailing.insert(trailing.begin(), text.back()); + text.pop_back(); + } + if (!phonemizer_) { + phonemizer_ = std::make_unique( + std::filesystem::path{espeak_.library_path}, + std::filesystem::path{espeak_.data_path}, + std::vector{espeak_.lang.empty() ? "en-us" : espeak_.lang}); + } + if (!text.empty()) { + bool all_space = true; + for (const char c : text) { + if (c != ' ') { all_space = false; break; } + } + if (all_space) { + for (const auto & ch : split_utf8(text)) emit({ch}, ids); + } else { + emit(split_utf8(phonemizer_->phonemize(text, 2)), ids); + } + } + for (const auto & ch : split_utf8(trailing)) emit({ch}, ids); + continue; + } + if (part.lang != 'z') continue; // "other"-only segments are dropped + + // Chinese segment: digit normalization, then jieba word segmentation + // (the model-local MixSegment port, identical to python jieba.cut). + // Each word is one reading + sandhi unit, exactly like the reference + // pipeline lazy_pinyin(jieba.cut(text), tone_sandhi=True). + const std::string normalized = text::normalize_chinese_text(part.text); + std::vector words; + { + std::lock_guard lock(segmenter_mutex_); + if (!segmenter_) { + if (!std::filesystem::is_regular_file(tables_.jieba_dict) || + !std::filesystem::is_regular_file(tables_.hmm_model)) { + throw std::runtime_error( + "zipvoice: jieba dictionaries missing (expected zh_jieba_dict.txt " + "and zh_hmm_model.txt in the package or next to it); run " + "tools/community_models/export_zipvoice_zh_dict.py"); + } + segmenter_ = std::make_unique( + tables_.jieba_dict, tables_.hmm_model); + } + words = segmenter_->cut(normalized); + } + for (const auto & word : words) { + const auto word_chars = split_utf8(word); + bool has_han = false; + for (const auto & ch : word_chars) { + if (is_han_char(utf8_to_codepoint(ch))) has_han = true; + } + if (!has_han) { + for (const auto & ch : word_chars) emit({ch}, ids); + continue; + } + std::vector tokens; + if (const auto it = phrases_.find(word); it != phrases_.end()) { + tokens = it->second; + } else { + // Unknown word: per-character default readings; characters + // without a pinyin entry are skipped, like the reference. + for (const auto & ch : word_chars) { + if (const auto cit = chars_.find(ch); cit != chars_.end()) { + tokens.insert(tokens.end(), cit->second.begin(), cit->second.end()); + } + } + } + apply_tone_sandhi(tokens, word); + emit(tokens, ids); + } + } + return ids; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/jieba_segmenter.cpp b/src/community_models/zipvoice/jieba_segmenter.cpp new file mode 100644 index 000000000..860ce3f35 --- /dev/null +++ b/src/community_models/zipvoice/jieba_segmenter.cpp @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: MIT +// +// Focused port of cppjieba's MixSegment for ZipVoice. Derived from: +// cppjieba, Copyright (c) 2013 Yanyi Wu (MIT) +// jieba, Copyright (c) 2012 Sun Junyi (MIT) +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Only the pieces the ZipVoice Chinese frontend needs are kept: the PreFilter +// separator pass, the maximum-probability DAG segmentation over the jieba +// dictionary, and the four-state BMES HMM used for runs of dictionary-missed +// single characters. The dictionary files are model resources embedded in the +// GGUF, not a vendored library. + +#include "jieba_segmenter.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +// cppjieba Utils.hpp MIN_DOUBLE, used as the "impossible" log-probability. +constexpr double kMinDouble = -3.14e+100; + +// cppjieba HMMModel::STATUS_SUM order: 0=B, 1=E, 2=M, 3=S. +constexpr size_t kStateE = 1; +constexpr size_t kStateS = 3; + +// cppjieba Utils.hpp Trim. +void trim(std::string & s) { + auto not_space = [](unsigned char c) { return !std::isspace(c); }; + s.erase(s.begin(), std::find_if(s.begin(), s.end(), not_space)); + s.erase(std::find_if(s.rbegin(), s.rend(), not_space).base(), s.end()); +} + +size_t count_codepoints(const std::string & s) { + size_t n = 0; + for (unsigned char c : s) { + if ((c & 0xC0) != 0x80) ++n; + } + return n; +} + +} // namespace + +JiebaSegmenter::JiebaSegmenter(const std::filesystem::path & dict_path, + const std::filesystem::path & hmm_path) { + // --- dictionary: , weight = log(freq / freq_sum) --- + std::ifstream dict(dict_path); + if (!dict) { + throw std::runtime_error("zipvoice: cannot open jieba dictionary " + dict_path.string()); + } + std::vector> entries; + entries.reserve(400000); + double freq_sum = 0.0; + size_t max_len = 1; + std::string line; + while (std::getline(dict, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + const size_t first = line.find(' '); + if (first == std::string::npos) continue; + const size_t second = line.find(' ', first + 1); + if (second == std::string::npos) continue; + std::string word = line.substr(0, first); + const double freq = std::strtod(line.c_str() + first + 1, nullptr); + freq_sum += freq; + max_len = std::max(max_len, count_codepoints(word)); + entries.emplace_back(std::move(word), freq); + } + if (entries.empty() || !(freq_sum > 0.0)) { + throw std::runtime_error("zipvoice: empty jieba dictionary " + dict_path.string()); + } + weights_.reserve(entries.size() * 2); + double min_weight = std::numeric_limits::infinity(); + for (const auto & [word, freq] : entries) { + const double weight = std::log(freq / freq_sum); + weights_[word] = weight; + min_weight = std::min(min_weight, weight); + } + min_weight_ = min_weight; + max_word_len_ = max_len; + + // --- HMM: start/trans/emit tables (cppjieba hmm_model.utf8) --- + std::ifstream hmm(hmm_path); + if (!hmm) { + throw std::runtime_error("zipvoice: cannot open jieba HMM model " + hmm_path.string()); + } + const auto next_content_line = [&](std::string & out) { + while (std::getline(hmm, out)) { + trim(out); + if (!out.empty() && out[0] != '#') return true; + } + return false; + }; + const auto parse_doubles = [](const std::string & text, double * out, size_t count) { + std::istringstream stream(text); + for (size_t i = 0; i < count; ++i) { + if (!(stream >> out[i])) { + throw std::runtime_error("zipvoice: malformed jieba HMM model"); + } + } + }; + if (!next_content_line(line)) throw std::runtime_error("zipvoice: truncated jieba HMM model"); + parse_doubles(line, start_prob_, 4); + for (size_t i = 0; i < 4; ++i) { + if (!next_content_line(line)) throw std::runtime_error("zipvoice: truncated jieba HMM model"); + parse_doubles(line, trans_prob_[i], 4); + } + for (size_t state = 0; state < 4; ++state) { + if (!next_content_line(line)) throw std::runtime_error("zipvoice: truncated jieba HMM model"); + size_t pos = 0; + while (pos < line.size()) { + const size_t comma = line.find(',', pos); + const std::string entry = line.substr( + pos, comma == std::string::npos ? std::string::npos : comma - pos); + const size_t colon = entry.find(':'); + if (colon != std::string::npos && colon > 0) { + const auto runes = decode(entry.substr(0, colon)); + if (!runes.empty()) { + emit_prob_[state][runes.front().cp] = + std::strtod(entry.c_str() + colon + 1, nullptr); + } + } + if (comma == std::string::npos) break; + pos = comma + 1; + } + } +} + +std::vector JiebaSegmenter::decode(const std::string & text) { + std::vector runes; + runes.reserve(text.size() / 2 + 1); + for (size_t i = 0; i < text.size();) { + const auto lead = static_cast(text[i]); + uint32_t cp = lead; + uint32_t len = 1; + if (lead < 0x80) { + cp = lead; + } else if ((lead & 0xE0) == 0xC0 && i + 1 < text.size()) { + cp = (uint32_t(lead & 0x1F) << 6) | + (uint32_t(static_cast(text[i + 1])) & 0x3F); + len = 2; + } else if ((lead & 0xF0) == 0xE0 && i + 2 < text.size()) { + cp = (uint32_t(lead & 0x0F) << 12) | + (uint32_t(static_cast(text[i + 1])) & 0x3F) << 6 | + (uint32_t(static_cast(text[i + 2])) & 0x3F); + len = 3; + } else if ((lead & 0xF8) == 0xF0 && i + 3 < text.size()) { + cp = (uint32_t(lead & 0x07) << 18) | + (uint32_t(static_cast(text[i + 1])) & 0x3F) << 12 | + (uint32_t(static_cast(text[i + 2])) & 0x3F) << 6 | + (uint32_t(static_cast(text[i + 3])) & 0x3F); + len = 4; + } + runes.push_back({cp, static_cast(i), len}); + i += len; + } + return runes; +} + +std::string JiebaSegmenter::slice(const std::string & text, const std::vector & runes, + size_t begin, size_t end_inclusive) { + const size_t start = runes[begin].offset; + const size_t end = end_inclusive + 1 < runes.size() + ? runes[end_inclusive + 1].offset + : text.size(); + return text.substr(start, end - start); +} + +bool JiebaSegmenter::is_separator(uint32_t cp) { + // cppjieba SegmentBase SPECIAL_SEPARATORS: " \t\n，。". + return cp == 0x20 || cp == 0x09 || cp == 0x0A || cp == 0xFF0C || cp == 0x3002; +} + +size_t JiebaSegmenter::sequential_letter_rule(const std::vector & runes, + size_t begin, size_t end) { + uint32_t x = runes[begin].cp; + if (!(('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z'))) return begin; + ++begin; + while (begin != end) { + x = runes[begin].cp; + if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) ++begin; + else break; + } + if (begin != end && runes[begin].cp == '.') { + if (begin + 1 != end && runes[begin + 1].cp >= '0' && runes[begin + 1].cp <= '9') { + ++begin; + while (begin != end && runes[begin].cp >= '0' && runes[begin].cp <= '9') ++begin; + } + } + return begin; +} + +size_t JiebaSegmenter::numbers_rule(const std::vector & runes, size_t begin, size_t end) { + uint32_t x = runes[begin].cp; + if (!('0' <= x && x <= '9')) return begin; + ++begin; + while (begin != end) { + x = runes[begin].cp; + if (('0' <= x && x <= '9') || ('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) ++begin; + else break; + } + if (begin != end && runes[begin].cp == '.') { + if (begin + 1 != end && runes[begin + 1].cp >= '0' && runes[begin + 1].cp <= '9') { + ++begin; + while (begin != end && runes[begin].cp >= '0' && runes[begin].cp <= '9') ++begin; + } + } + return begin; +} + +double JiebaSegmenter::emit_probability(size_t state, uint32_t cp) const { + const auto & table = emit_prob_[state]; + const auto it = table.find(cp); + return it == table.end() ? kMinDouble : it->second; +} + +void JiebaSegmenter::viterbi_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const { + const size_t X = end - begin; + if (X == 0) return; + std::vector weight(4 * X); + std::vector path(4 * X); + for (size_t y = 0; y < 4; ++y) { + weight[y * X] = start_prob_[y] + emit_probability(y, runes[begin].cp); + path[y * X] = -1; + } + for (size_t x = 1; x < X; ++x) { + for (size_t y = 0; y < 4; ++y) { + const size_t now = x + y * X; + weight[now] = kMinDouble; + path[now] = static_cast(kStateE); + const double emit = emit_probability(y, runes[begin + x].cp); + for (size_t prev = 0; prev < 4; ++prev) { + const double candidate = weight[x - 1 + prev * X] + trans_prob_[prev][y] + emit; + if (candidate > weight[now]) { + weight[now] = candidate; + path[now] = static_cast(prev); + } + } + } + } + const double end_e = weight[(X - 1) + kStateE * X]; + const double end_s = weight[(X - 1) + kStateS * X]; + int state = end_e >= end_s ? static_cast(kStateE) : static_cast(kStateS); + std::vector status(X); + for (size_t x = X; x-- > 0;) { + status[x] = state; + state = path[x + static_cast(state) * X]; + } + size_t left = begin; + for (size_t i = 0; i < X; ++i) { + if (status[i] % 2 != 0) { // E or S ends a word + words.push_back(slice(text, runes, left, begin + i)); + left = begin + i + 1; + } + } +} + +void JiebaSegmenter::hmm_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const { + size_t left = begin; + size_t right = begin; + while (right < end) { + if (runes[right].cp < 0x80) { + if (left != right) viterbi_cut(text, runes, left, right, words); + left = right; + size_t next = sequential_letter_rule(runes, left, end); + if (next == left) next = numbers_rule(runes, left, end); + if (next == left) next = left + 1; + right = next; + words.push_back(slice(text, runes, left, right - 1)); + left = right; + } else { + ++right; + } + } + if (left != right) viterbi_cut(text, runes, left, right, words); +} + +void JiebaSegmenter::segment_range(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const { + const size_t n = end - begin; + if (n == 0) return; + struct Candidate { + size_t end_index; + double weight; + }; + // MP DAG candidates: the single character is always a candidate (unknown + // characters fall back to the dictionary's minimum weight, like cppjieba); + // longer candidates exist only when the whole span is a dictionary word. + std::vector> nexts(n); + for (size_t k = 0; k < n; ++k) { + const size_t i = begin + k; + const std::string single = slice(text, runes, i, i); + const auto single_it = weights_.find(single); + nexts[k].push_back({k, single_it != weights_.end() ? single_it->second : min_weight_}); + for (size_t len = 2; len <= max_word_len_ && k + len <= n; ++len) { + const auto it = weights_.find(slice(text, runes, i, i + len - 1)); + if (it != weights_.end()) nexts[k].push_back({k + len - 1, it->second}); + } + } + // Maximum-probability route, right to left (cppjieba CalcDP). + std::vector best_weight(n + 1, 0.0); + std::vector best_len(n, 1); + for (size_t k = n; k-- > 0;) { + double best = kMinDouble; + size_t length = 1; + for (const auto & candidate : nexts[k]) { + const double value = candidate.weight + + (candidate.end_index + 1 < n ? best_weight[candidate.end_index + 1] : 0.0); + if (value > best) { + best = value; + length = candidate.end_index - k + 1; + } + } + best_weight[k] = best; + best_len[k] = length; + } + // MP words as rune ranges. + std::vector> mp_words; + for (size_t k = 0; k < n;) { + const size_t length = best_len[k]; + mp_words.push_back({begin + k, begin + k + length - 1}); + k += length; + } + // MixSegment: keep multi-character words; re-cut each run of consecutive + // single-character words with the HMM. + for (size_t r = 0; r < mp_words.size();) { + if (mp_words[r].first != mp_words[r].second) { + words.push_back(slice(text, runes, mp_words[r].first, mp_words[r].second)); + ++r; + continue; + } + size_t run_end = r; + while (run_end < mp_words.size() && mp_words[run_end].first == mp_words[run_end].second) { + ++run_end; + } + hmm_cut(text, runes, mp_words[r].first, mp_words[run_end - 1].first + 1, words); + r = run_end; + } +} + +std::vector JiebaSegmenter::cut(const std::string & text) const { + std::vector words; + const auto runes = decode(text); + if (runes.empty()) return words; + // PreFilter: separators are single-character words; the runs between them + // are segmented independently. + size_t i = 0; + while (i < runes.size()) { + if (is_separator(runes[i].cp)) { + words.push_back(slice(text, runes, i, i)); + ++i; + continue; + } + size_t j = i; + while (j < runes.size() && !is_separator(runes[j].cp)) ++j; + segment_range(text, runes, i, j, words); + i = j; + } + return words; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/jieba_segmenter.h b/src/community_models/zipvoice/jieba_segmenter.h new file mode 100644 index 000000000..941c3541e --- /dev/null +++ b/src/community_models/zipvoice/jieba_segmenter.h @@ -0,0 +1,67 @@ +#pragma once + +// SPDX-License-Identifier: MIT +// +// Minimal Jieba word segmentation for the ZipVoice Chinese frontend. +// +// This is a focused port of the parts of cppjieba (Copyright (c) 2013 Yanyi +// Wu, MIT) and jieba (Copyright (c) 2012 Sun Junyi, MIT) that the frontend +// needs: the PreFilter separator pass, the maximum-probability DAG +// segmentation over jieba.dict.utf8, and the four-state BMES HMM over +// hmm_model.utf8. It reads the same dictionary files that are embedded in the +// model GGUF, so no language-specific library is vendored. +// +// cppjieba: https://github.com/yanyiwu/cppjieba (MIT) +// jieba: https://github.com/fxsjy/jieba (MIT) + +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { + +// Word segmentation equivalent to cppjieba::MixSegment (HMM enabled), which in +// turn matches python jieba.cut for the same dictionaries. +class JiebaSegmenter { +public: + JiebaSegmenter(const std::filesystem::path & dict_path, + const std::filesystem::path & hmm_path); + + // Word boundaries for `text` (UTF-8). Separators become single-character + // words; runs of dictionary-missed single characters are re-segmented with + // the BMES HMM, exactly like MixSegment. + std::vector cut(const std::string & text) const; + +private: + struct Rune { + uint32_t cp = 0; + uint32_t offset = 0; // byte offset in the source string + uint32_t len = 0; // byte length + }; + + static std::vector decode(const std::string & text); + static std::string slice(const std::string & text, const std::vector & runes, + size_t begin, size_t end_inclusive); + static bool is_separator(uint32_t cp); + static size_t sequential_letter_rule(const std::vector & runes, size_t begin, size_t end); + static size_t numbers_rule(const std::vector & runes, size_t begin, size_t end); + // MP DAG + HMM over the rune range [begin, end). + void segment_range(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const; + void hmm_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const; + void viterbi_cut(const std::string & text, const std::vector & runes, + size_t begin, size_t end, std::vector & words) const; + double emit_probability(size_t state, uint32_t cp) const; + + std::unordered_map weights_; // word -> log(freq/freq_sum) + double min_weight_ = 0.0; + size_t max_word_len_ = 1; + double start_prob_[4] = {}; + double trans_prob_[4][4] = {}; + std::unordered_map emit_prob_[4]; +}; + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/session.cpp b/src/community_models/zipvoice/session.cpp new file mode 100644 index 000000000..87d52b4b8 --- /dev/null +++ b/src/community_models/zipvoice/session.cpp @@ -0,0 +1,278 @@ +#include "engine/community_models/zipvoice/session.h" + +#include "engine/community_models/zipvoice/synthesize.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/model_spec/package.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/text/chunking.h" +#include "engine/framework/runtime/spec_backed_model.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +constexpr const char * kFamily = "zipvoice"; + +struct ZipVoiceAssets { + assets::ResourceBundle resources; + std::filesystem::path checkpoint; // .gguf or safetensors dev dir + std::filesystem::path model_dir; + // session defaults (mutable only during session construction) + int num_steps = 8; + float guidance_scale = 3.0F; + float t_shift = 0.5F; +}; + +const runtime::AudioBuffer * reference_audio(const runtime::TaskRequest & request) { + if (request.voice.has_value() && + request.voice->speaker.has_value() && + request.voice->speaker->audio.has_value()) { + return &*request.voice->speaker->audio; + } + return request.audio_input.has_value() ? &*request.audio_input : nullptr; +} + +std::filesystem::path find_checkpoint(const std::filesystem::path & model_path) { + namespace fs = std::filesystem; + if (fs::is_regular_file(model_path)) { + return model_path; + } + std::vector ggufs, safetensors; + for (const auto & entry : fs::directory_iterator(model_path)) { + const auto ext = entry.path().extension(); + // skip the vocoder package when it is staged in the same directory + if (ext == ".gguf") ggufs.push_back(entry.path()); + else if (ext == ".safetensors") safetensors.push_back(entry.path()); + } + std::sort(ggufs.begin(), ggufs.end()); + std::sort(safetensors.begin(), safetensors.end()); + if (!ggufs.empty()) return ggufs.back(); + if (!safetensors.empty()) return safetensors.back(); + throw std::runtime_error( + "zipvoice: no .gguf/.safetensors checkpoint found in " + model_path.string()); +} + +bool tensor_file_has_namespace(const std::filesystem::path & path, const char * namespace_name) { + try { + const auto source = assets::open_tensor_source(path); + const std::string prefix = std::string(namespace_name) + "/backbone.embed.weight"; + const std::string prefix_dots = std::string(namespace_name) + ".backbone.embed.weight"; + return source->has_tensor(prefix) || source->has_tensor(prefix_dots); + } catch (...) { + return false; + } +} + +std::optional find_tensor_file(const std::filesystem::path & dir) { + namespace fs = std::filesystem; + if (!fs::is_directory(dir)) return std::nullopt; + for (const char * ext : {".safetensors", ".gguf"}) { + for (const auto & entry : fs::directory_iterator(dir)) { + if (entry.path().extension() == ext) return entry.path(); + } + } + return std::nullopt; +} + +std::shared_ptr load_assets(const std::filesystem::path & model_path) { + auto holder = std::make_shared(); + // Spec-resolved bundle: registered sidecars (tokens, model_config and the + // optional zh_* frontend tables) resolve to materialized GGUF-embedded + // copies or development-directory files; audio.cpp community models + // (audio8_asr, vibeasr, glm_tts, ...) follow the same pattern. + holder->resources = engine::model_spec::load_resource_bundle_for_family(model_path, kFamily); + holder->checkpoint = find_checkpoint(model_path); + holder->model_dir = + std::filesystem::is_regular_file(model_path) ? model_path.parent_path() : model_path; + return holder; +} + +class ZipVoiceSession final : public runtime::IOfflineVoiceTaskSession { +public: + ZipVoiceSession( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) + : task_kind_(task.task), + run_mode_(task.mode), + assets_(std::move(assets)), + contract_(std::move(contract)) { + if (assets_ == nullptr) { + throw std::runtime_error("zipvoice session requires assets"); + } + if (contract_ == nullptr) { + throw std::runtime_error("zipvoice session requires a model contract"); + } + runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "ZipVoice"); + // Vocos resolution order (f5_tts pattern): + // 1. zipvoice.vocos_path session option + // 2. bundled "vocos" namespace in the GGUF checkpoint + // 3. vocos.safetensors next to the checkpoint + // 4. vocos-mel-24khz package next to the model directory + namespace fs = std::filesystem; + if (const auto v = runtime::find_option(options.options, {"zipvoice.vocos_path", "vocos_path"})) { + vocos_path_ = *v; + } else if (assets_->checkpoint.extension() == ".gguf" && + tensor_file_has_namespace(assets_->checkpoint, "vocos")) { + vocos_path_ = assets_->checkpoint.string(); + } else { + const fs::path sibling = assets_->checkpoint.parent_path() / "vocos.safetensors"; + if (fs::exists(sibling)) { + vocos_path_ = sibling.string(); + } else if (const auto pkg = find_tensor_file( + assets_->model_dir.parent_path() / "vocos-mel-24khz")) { + vocos_path_ = pkg->string(); + } + } + if (vocos_path_.empty()) { + throw std::runtime_error( + "zipvoice: no vocos vocoder found; install the vocos-mel-24khz " + "package or set session option zipvoice.vocos_path"); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.num_inference_steps", "num_inference_steps"})) { + num_steps_ = std::stoi(*v); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.guidance_scale", "guidance_scale"})) { + guidance_scale_ = std::stof(*v); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.t_shift", "t_shift"})) { + t_shift_ = std::stof(*v); + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.espeak_data_path", "espeak_data_path"})) { + espeak_data_path_ = *v; + } + if (const auto v = runtime::find_option(options.options, {"zipvoice.espeak_library_path", "espeak_library_path"})) { + espeak_library_path_ = *v; + } + device_.backend_type = options.backend.type; + device_.device_index = options.backend.device; + device_.threads = options.backend.threads; + } + + std::string family() const noexcept override { return kFamily; } + runtime::VoiceTaskKind task_kind() const noexcept override { return task_kind_; } + runtime::RunMode run_mode() const noexcept override { return run_mode_; } + + void prepare(const runtime::SessionPreparationRequest & request) override { + (void) request; + } + + runtime::TaskResult run(const runtime::TaskRequest & request) override { + const auto budget = text::parse_text_chunk_size_override(request.options).value_or(128); + if (budget <= 0) throw std::invalid_argument("zipvoice: text_chunk_size must be positive"); + const auto mode = text::parse_text_chunk_mode_override(request.options).value_or(text::TextChunkMode::Default); + const auto chunks = runtime::chunk_text_request(request, budget, mode); + runtime::AudioBuffer merged; + for (const auto & chunk : chunks) { + auto result = run_chunk(chunk); + runtime::append_audio_buffer(merged, *result.audio_output); + } + if (merged.samples.empty()) throw std::invalid_argument("zipvoice: empty text chunks"); + runtime::TaskResult result; + result.audio_output = std::move(merged); + return result; + } + + runtime::TaskResult run_chunk(const runtime::TaskRequest & request) { + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("zipvoice requires input text"); + } + const runtime::AudioBuffer * ref = reference_audio(request); + if (ref == nullptr || ref->samples.empty()) { + throw std::runtime_error( + "zipvoice requires reference voice audio (voice preset or voice_ref)"); + } + const auto ref_text_it = request.options.find("reference_text"); + if (ref_text_it == request.options.end() || ref_text_it->second.empty()) { + throw std::runtime_error( + "zipvoice requires reference_text (transcript of the reference audio)"); + } + + ZipVoiceSynthesisRequest req; + req.text = request.text_input->text; + req.ref_text = ref_text_it->second; + req.ref_audio = ref->samples; + req.ref_sample_rate = ref->sample_rate; + req.ref_channels = ref->channels; + req.tokenizer = "emilia"; // fixed frontend: zh/en/mixed via the Emilia pipeline + req.espeak_library_path = espeak_library_path_; + req.espeak_data_path = espeak_data_path_; + if (const auto v = runtime::find_option(request.options, {"lang"})) req.lang = *v; + req.num_steps = num_steps_; + req.guidance_scale = guidance_scale_; + req.t_shift = t_shift_; + if (const auto v = runtime::find_option(request.options, {"num_inference_steps"})) { + req.num_steps = std::stoi(*v); + } + if (const auto v = runtime::find_option(request.options, {"guidance_scale", "cfg_strength"})) { + req.guidance_scale = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"t_shift"})) { + req.t_shift = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"speed"})) { + req.speed = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"feat_scale"})) { + req.feat_scale = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"target_rms"})) { + req.target_rms = std::stof(*v); + } + if (const auto v = runtime::find_option(request.options, {"seed"})) { + req.seed = static_cast(std::stoul(*v)); + req.fixed_seed = true; + } + + auto out = zipvoice_synthesize( + assets_->checkpoint.string(), vocos_path_, req, device_, &assets_->resources); + + runtime::TaskResult result; + runtime::AudioBuffer audio; + audio.sample_rate = out.sample_rate; + audio.channels = 1; + audio.samples = std::move(out.audio); + result.audio_output = std::move(audio); + return result; + } + +private: + runtime::VoiceTaskKind task_kind_; + runtime::RunMode run_mode_; + std::shared_ptr assets_; + std::shared_ptr contract_; + std::string vocos_path_; + int num_steps_ = 8; + float guidance_scale_ = 3.0F; + float t_shift_ = 0.5F; + std::string espeak_library_path_; + std::string espeak_data_path_; + ZipVoiceComputeDevice device_; +}; + +} // namespace + +std::shared_ptr make_zipvoice_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = std::string(kFamily); + config.load_assets = load_assets; + config.create_session = []( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, options, std::move(assets), std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/synthesize.cpp b/src/community_models/zipvoice/synthesize.cpp new file mode 100644 index 000000000..bb5a33ff5 --- /dev/null +++ b/src/community_models/zipvoice/synthesize.cpp @@ -0,0 +1,1030 @@ +#include "engine/community_models/zipvoice/synthesize.h" +#include "engine/framework/runtime/cache_slots.h" +#include "engine/framework/modules/vocoders/vocos_vocoder.h" + +#include "engine/community_models/zipvoice/emilia_tokenizer.h" +#include "engine/community_models/zipvoice/weights.h" +#include "engine/community_models/zipvoice/zipformer.h" + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/audio/espeak_phonemizer.h" +#include "engine/framework/audio/conversion.h" + +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +// --------------------------------------------------------------------------- +// log-mel (VocosFbank parity: torchaudio MelSpectrogram 24k/1024/256/100, +// power=1, htk scale, f_max=12000, log clamp 1e-7, lhotse frame alignment) + +constexpr float kPi = 3.14159265358979323846F; +constexpr int kSampleRate = 24000; +constexpr int kNfft = 1024; +constexpr int kHop = 256; +constexpr int kNMel = 100; +constexpr float kFMin = 0.0F; +constexpr float kFMax = 12000.0F; + +float hz_to_mel_htk(float hz) { return 2595.0F * std::log10(1.0F + hz / 700.0F); } +float mel_to_hz_htk(float mel) { return 700.0F * (std::pow(10.0F, mel / 2595.0F) - 1.0F); } + +std::vector mel_filterbank() { + std::vector fb; + { + const int n_freqs = kNfft / 2 + 1; + const float m_min = hz_to_mel_htk(kFMin); + const float m_max = hz_to_mel_htk(kFMax); + std::vector mels(kNMel + 2); + for (int i = 0; i < kNMel + 2; ++i) { + mels[i] = mel_to_hz_htk(m_min + (m_max - m_min) * i / (kNMel + 1)); + } + fb.assign(static_cast(n_freqs) * kNMel, 0.0F); + for (int m = 0; m < kNMel; ++m) { + const float lo = mels[m]; + const float mid = mels[m + 1]; + const float hi = mels[m + 2]; + for (int f = 0; f < n_freqs; ++f) { + const float freq = static_cast(f) * kSampleRate / kNfft; + if (freq <= lo || freq >= hi) continue; + const float w = freq <= mid + ? (freq - lo) / (mid - lo) + : (hi - freq) / (hi - mid); + fb[static_cast(f) * kNMel + m] = w; + } + } + } + return fb; +} + +void fft_inplace(std::vector & re, std::vector & im, bool inverse) { + const size_t n = re.size(); + for (size_t i = 1, j = 0; i < n; ++i) { + size_t bit = n >> 1; + for (; j & bit; bit >>= 1) j ^= bit; + j ^= bit; + if (i < j) { + std::swap(re[i], re[j]); + std::swap(im[i], im[j]); + } + } + for (size_t len = 2; len <= n; len <<= 1) { + const float ang = static_cast(2.0 * kPi / static_cast(len)) * (inverse ? 1 : -1); + for (size_t i = 0; i < n; i += len) { + for (size_t k = 0; k < len / 2; ++k) { + const float wr = std::cos(ang * static_cast(k)); + const float wi = std::sin(ang * static_cast(k)); + const size_t a = i + k; + const size_t b = i + k + len / 2; + const float vr = re[b] * wr - im[b] * wi; + const float vi = re[b] * wi + im[b] * wr; + const float ur = re[a]; + const float ui = im[a]; + re[a] = ur + vr; + im[a] = ui + vi; + re[b] = ur - vr; + im[b] = ui - vi; + } + } + } + if (inverse) { + for (size_t i = 0; i < n; ++i) { + re[i] /= static_cast(n); + im[i] /= static_cast(n); + } + } +} + +} // namespace + +std::vector zipvoice_logmel(const std::vector & wav) { + if (wav.size() <= kNfft / 2) { + throw std::invalid_argument("zipvoice: reference audio must contain more than 512 samples at 24 kHz"); + } + const int n_freqs = kNfft / 2 + 1; + // lhotse compute_num_frames: (samples + hop/2) / hop + const int frames = std::max(1, (static_cast(wav.size()) + kHop / 2) / kHop); + std::vector hann(kNfft); + for (int i = 0; i < kNfft; ++i) { + hann[i] = 0.5F * (1.0F - std::cos(2.0F * static_cast(kPi) * i / kNfft)); + } + const auto & fb = mel_filterbank(); + std::vector mel(static_cast(frames) * kNMel); // [t, mel] row-major + std::vector re(kNfft), im(kNfft), mag(n_freqs); + for (int t = 0; t < frames; ++t) { + std::fill(re.begin(), re.end(), 0.0F); + std::fill(im.begin(), im.end(), 0.0F); + const int start = (t - 2) * kHop; // torchaudio center=True alignment + for (int i = 0; i < kNfft; ++i) { + int r = start + i; + if (r < 0) r = -r; + if (r >= static_cast(wav.size())) r = 2 * static_cast(wav.size()) - 2 - r; + r = std::clamp(r, 0, static_cast(wav.size()) - 1); + re[i] = wav[static_cast(r)] * hann[i]; + } + fft_inplace(re, im, false); + for (int f = 0; f < n_freqs; ++f) { + mag[f] = std::sqrt(re[f] * re[f] + im[f] * im[f]); + } + for (int m = 0; m < kNMel; ++m) { + float acc = 0.0F; + for (int f = 0; f < n_freqs; ++f) { + acc += fb[static_cast(f) * kNMel + m] * mag[f]; + } + mel[static_cast(t) * kNMel + m] = std::log(std::max(acc, 1e-7F)); + } + } + return mel; +} + +std::vector zipvoice_vocos_decode(const std::string & path, const std::vector & mel) { + const auto free_backend = [](ggml_backend_t b) { ggml_backend_free(b); }; + std::unique_ptr backend( + core::init_backend({core::BackendType::Cpu, 0, 4}), free_backend); + modules::VocosVocoder vocos(path, backend.get(), 4); + return vocos.decode(mel); +} + +namespace { + +// --------------------------------------------------------------------------- +// Session/device-owned weights and bounded runtime graph slots + +std::unordered_map load_tokens(const std::filesystem::path & path); + +struct LoadedModel { + std::string path; + ZipVoiceConfig config; + ZipVoiceWeights weights; + ggml_backend_t backend = nullptr; + bool owns_backend = true; // false when the backend was injected by the caller + core::BackendType backend_type = core::BackendType::Cpu; // concrete resolved type + std::mutex mutex; // serializes graph use (graphs are not thread-safe) + runtime::CacheSlots> text_graphs{1}; + struct FmKey { + int64_t t; + int64_t b; + bool guidance; + bool operator==(const FmKey & o) const { return t == o.t && b == o.b && guidance == o.guidance; } + }; + runtime::CacheSlots> fm_graphs{1}; + runtime::CacheSlots, std::unique_ptr> tokenizers{1}; + std::unique_ptr vocos_graph; + std::string vocos_path; + ~LoadedModel() { + vocos_graph.reset(); + text_graphs.clear(); + fm_graphs.clear(); + weights = {}; + if (owns_backend) { + ggml_backend_free(backend); + } + } +}; + +std::vector decode_vocos(LoadedModel & model, const std::string & path, + const std::vector & mel, int threads) { + if (!model.vocos_graph || model.vocos_path != path) { + model.vocos_graph.reset(); + model.vocos_graph = std::make_unique(path, model.backend, threads); + model.vocos_path = path; + } + return model.vocos_graph->decode(mel); +} + +// Sidecar resolution: spec-registered resources (GGUF-embedded sidecars +// materialized by the framework, or files found in the development directory) +// take priority; loose files next to the checkpoint remain the fallback for +// direct API entry points (parity harnesses) that pass no bundle. +struct SidecarResolver { + const engine::assets::ResourceBundle * resources = nullptr; + + std::filesystem::path path(std::string_view id, const std::filesystem::path & dir, + std::string_view filename) const { + if (resources != nullptr) { + if (const auto * p = resources->find_file(id)) return *p; + } + return dir / filename; + } +}; + +// Resolve the requested backend: an injected backend wins; otherwise the +// device's backend type (BestAvailable by default = GPU first, CPU fallback). +core::BackendType resolve_backend_type(const ZipVoiceComputeDevice & device) { + if (device.backend != nullptr) { + return core::backend_type(device.backend); + } + return device.backend_type; +} + +} // namespace + +class ZipVoiceRuntimeState { +public: + std::mutex mutex; + std::shared_ptr model; + std::string key; +}; + +std::shared_ptr make_zipvoice_runtime_state() { + return std::make_shared(); +} + +namespace { +std::shared_ptr load_model( + const std::string & path, const SidecarResolver & sidecars, + const ZipVoiceComputeDevice & device) { + if (!device.runtime) throw std::invalid_argument("zipvoice: runtime is null"); + auto & state = *device.runtime; + std::lock_guard lock(state.mutex); + const auto cache_config_dir = std::filesystem::is_directory(path) + ? std::filesystem::path(path) : std::filesystem::path(path).parent_path(); + const std::string key = path + "::" + sidecars.path("model_config", cache_config_dir, "model.json").string() + + "::" + sidecars.path("tokens", cache_config_dir, "tokens.txt").string() + + "::" + std::to_string(static_cast(resolve_backend_type(device))) + + "::" + std::to_string(device.device_index) + "::" + std::to_string(reinterpret_cast(device.backend)); + if (state.model && state.key == key) return state.model; + state.model.reset(); + auto model = std::make_unique(); + model->path = path; + namespace fs = std::filesystem; + const fs::path p(path); + std::string prefix = ""; + std::shared_ptr source; + if (p.extension() == ".gguf") { + source = engine::assets::make_prefixed_tensor_source( + engine::assets::open_tensor_source(p), "model"); + } else { + // development layout: directory with -orig.safetensors + std::vector candidates; + if (fs::is_regular_file(p) && p.extension() == ".safetensors") { + candidates.push_back(p); + } else if (fs::is_directory(p)) { + for (const auto & entry : fs::directory_iterator(p)) { + if (entry.path().extension() == ".safetensors" && + entry.path().filename().string().find("vocos") == std::string::npos) { + candidates.push_back(entry.path()); + } + } + } + if (candidates.empty()) { + throw std::runtime_error("zipvoice: no safetensors checkpoint in " + path); + } + std::sort(candidates.begin(), candidates.end()); + source = engine::assets::open_tensor_source(candidates.back()); + } + const fs::path config_dir = fs::is_directory(p) ? p : p.parent_path(); + const auto config_path = sidecars.path("model_config", config_dir, "model.json"); + model->config = load_zipvoice_config(config_dir, source.get(), &config_path); + // vocab size from the embedding itself + model->config.vocab_size = static_cast( + source->require_metadata("embed.weight").shape[0]); + // pad id from tokens.txt (the "_" entry per the reference tokenizers) + { + const auto vocab = load_tokens(sidecars.path("tokens", config_dir, "tokens.txt")); + const auto it = vocab.find("_"); + if (it != vocab.end()) { + model->config.pad_id = it->second; + } + } + // Device-selected backend: a caller-injected backend (borrowed, not + // owned) or one created here — CPU compute with n_threads, or a GPU + // backend via BestAvailable (Metal on Apple Silicon, CUDA on NVIDIA). + if (device.backend != nullptr) { + model->backend = device.backend; + model->owns_backend = false; + } else { + core::BackendConfig cfg; + cfg.type = device.backend_type; + cfg.device = device.device_index; + cfg.threads = device.threads > 0 + ? device.threads + : static_cast(std::thread::hardware_concurrency()); + model->backend = core::init_backend(cfg); + model->owns_backend = true; + } + if (model->backend == nullptr) { + throw std::runtime_error("zipvoice: backend init failed"); + } + // BestAvailable resolves to a concrete backend (Metal on Apple + // Silicon); downstream allocation and vocoder decisions use it. + model->backend_type = core::backend_type(model->backend); + model->weights = load_zipvoice_weights( + *source, "", model->config, model->backend, model->backend_type); + + state.key = key; + state.model = std::move(model); + return state.model; +} + +// timestep_embedding host copy (matches zipvoice.modules.timestep_embedding) +std::vector timestep_embedding(float t, int64_t dim) { + const int64_t half = dim / 2; + std::vector out(static_cast(dim)); + for (int64_t i = 0; i < half; ++i) { + const float freq = std::exp(-std::log(10000.0F) * static_cast(i) / + static_cast(half)); + const float arg = t * freq; + out[static_cast(i)] = std::cos(arg); + out[static_cast(half + i)] = std::sin(arg); + } + return out; +} + +void upload_leaf(ggml_backend_t backend, ggml_tensor * leaf, const void * data, size_t bytes) { + (void)backend; + ggml_backend_tensor_set(leaf, data, 0, bytes); // void return; CPU sync path +} + +int compute_threads(const ZipVoiceComputeDevice & device) { + return device.threads > 0 ? device.threads + : static_cast(std::thread::hardware_concurrency()); +} + +// Fill per-stack pad bias / conv gate leaves given the valid frame count in +// the FULL-resolution sequence (frames >= valid are padding). +void fill_masks( + ggml_backend_t backend, + const ZipVoiceConfig & config, + FmDecoderGraph & g, + int64_t valid_frames) { + for (size_t s = 0; s < config.fm_downsampling_factor.size(); ++s) { + const int64_t ds = config.fm_downsampling_factor[s]; + // stack s runs at ceil(T / ds) frames (factors are relative to the + // full frame rate); the ::ds-sampled mask keeps ceil(valid / ds) + // valid frames. + const int64_t T_s = (g.T + ds - 1) / ds; + const int64_t valid_s = std::min((valid_frames + ds - 1) / ds, T_s); + std::vector bias(static_cast(T_s)); + std::vector gate(static_cast(T_s)); + for (int64_t i = 0; i < T_s; ++i) { + const bool pad = i >= valid_s; + bias[static_cast(i)] = pad ? -1000.0F : 0.0F; + gate[static_cast(i)] = pad ? 0.0F : 1.0F; + } + upload_leaf(backend, g.pad_bias[s], bias.data(), bias.size() * sizeof(float)); + upload_leaf(backend, g.conv_gate[s], gate.data(), gate.size() * sizeof(float)); + } +} + +struct VelocityInputs { + // xt, text_condition, speech_condition: [T, F] row-major (t, f) + std::vector xt, text_condition, speech_condition; +}; + +// Run one fm_decoder evaluation. B = 2 duplicates every input (CFG halves +// are prepared by the caller through `halves`). +std::vector run_velocity( + LoadedModel & model, + const ZipVoiceComputeDevice & device, + int64_t T, + int64_t B, + const float * xt, // [B, T, F] + const float * text_cond, // [B, T, F] + const float * speech_cond, // [B, T, F] + int64_t valid_frames, + float t_value, + const float * guidance_embedding, // nullptr for base model + float * scratch_time = nullptr) { + const auto & config = model.config; + const bool with_guidance = guidance_embedding != nullptr; + std::lock_guard lock(model.mutex); + FmDecoderGraph * g = nullptr; + const LoadedModel::FmKey key{T, B, with_guidance}; + if (const auto * slot = model.fm_graphs.find(key)) { + g = slot->get(); + } else { + model.fm_graphs.clear(); + auto built = build_fm_decoder_graph( + model.weights, config, T, B, with_guidance, false, model.backend); + model.fm_graphs.put(key, std::make_unique(std::move(built))); + g = model.fm_graphs.find(key)->get(); + } + + // x_cat [3F, T, B]: interleave channels (xt | text | speech) + const int64_t F = config.feat_dim; + std::vector x_cat(static_cast(3 * F * T * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t i = 0; i < T; ++i) { + float * dst = x_cat.data() + static_cast(3 * F * (i + b * T)); + const float * x_src = xt + static_cast((i + b * T) * F); + const float * t_src = text_cond + static_cast((i + b * T) * F); + const float * s_src = speech_cond + static_cast((i + b * T) * F); + std::memcpy(dst, x_src, F * sizeof(float)); + std::memcpy(dst + F, t_src, F * sizeof(float)); + std::memcpy(dst + 2 * F, s_src, F * sizeof(float)); + } + } + upload_leaf(model.backend, g->x_cat, x_cat.data(), x_cat.size() * sizeof(float)); + const auto time_values = scratch_time != nullptr + ? std::vector(scratch_time, scratch_time + config.time_embed_dim) + : timestep_embedding(t_value, config.time_embed_dim); + upload_leaf(model.backend, g->time_emb, time_values.data(), + static_cast(config.time_embed_dim) * sizeof(float)); + if (with_guidance) { + upload_leaf(model.backend, g->guidance_emb, guidance_embedding, + static_cast(config.time_embed_dim) * sizeof(float)); + } + fill_masks(model.backend, config, *g, valid_frames); + + core::set_backend_threads(model.backend, compute_threads(device)); + const auto status = core::compute_backend_graph(model.backend, g->graph, nullptr, "zipvoice_fm"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("zipvoice: fm decoder graph compute failed"); + } + ggml_backend_synchronize(model.backend); + + // output [F, T, B] -> [B, T, F] + std::vector out_vec(ggml_nelements(g->output)); + ggml_backend_tensor_get(g->output, out_vec.data(), 0, out_vec.size() * sizeof(float)); + const float * out = out_vec.data(); + std::vector v(static_cast(T * F * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t i = 0; i < T; ++i) { + for (int64_t f = 0; f < F; ++f) { + v[static_cast((i + b * T) * F + f)] = + out[static_cast(f + i * F + b * T * F)]; + } + } + } + return v; +} + +// forward_text_inference_ratio_duration (single utterance) +struct TextConditionResult { + std::vector condition; // [T, F] + int64_t T = 0; +}; + +TextConditionResult compute_text_condition( + LoadedModel & model, + const ZipVoiceComputeDevice & device, + const std::vector & tokens, // target tokens + const std::vector & prompt_tokens, // prompt tokens + int64_t prompt_features_len, + float speed) { + // pad_labels appends one trailing pad token; the original token count + // defines the attention and convolution masks. + // NOTE: tokens_lens in the reference EXCLUDES the pad token, so the + // duration split uses token_count, not S. + std::vector cat(prompt_tokens); + cat.insert(cat.end(), tokens.begin(), tokens.end()); + const int64_t token_count = static_cast(cat.size()); + if (!std::isfinite(speed) || speed <= 0 || prompt_features_len <= 0 || tokens.empty()) { + throw std::invalid_argument("zipvoice: nonempty tokens, positive prompt length and speed are required"); + } + for (const int32_t token : cat) { + if (token < 0 || token >= model.config.vocab_size) { + throw std::invalid_argument("zipvoice: token id out of vocabulary range"); + } + } + cat.push_back(model.config.pad_id); + const int64_t S = static_cast(cat.size()); + if (token_count == 0 || prompt_tokens.empty()) { + throw std::runtime_error("zipvoice: empty token stream"); + } + const int64_t features_len = prompt_features_len + + static_cast(std::ceil( + static_cast(prompt_features_len) / + static_cast(prompt_tokens.size()) * + static_cast(tokens.size()) / speed)); + const int64_t T = features_len; + + std::lock_guard lock(model.mutex); + TextEncoderGraph * g = nullptr; + if (const auto * slot = model.text_graphs.find(S)) { + g = slot->get(); + } else { + model.text_graphs.clear(); + auto built = build_text_encoder_graph(model.weights, model.config, S, false, model.backend); + model.text_graphs.put(S, std::make_unique(std::move(built))); + g = model.text_graphs.find(S)->get(); + } + upload_leaf(model.backend, g->token_ids, cat.data(), cat.size() * sizeof(int32_t)); + std::vector zeros(static_cast(S), 0.0F); + std::vector ones(static_cast(S), 1.0F); + zeros.back() = -1000.0F; + ones.back() = 0.0F; + upload_leaf(model.backend, g->pad_bias[0], zeros.data(), zeros.size() * sizeof(float)); + upload_leaf(model.backend, g->conv_gate[0], ones.data(), ones.size() * sizeof(float)); + + core::set_backend_threads(model.backend, compute_threads(device)); + const auto status = core::compute_backend_graph(model.backend, g->graph, nullptr, "zipvoice_text"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("zipvoice: text encoder graph compute failed"); + } + ggml_backend_synchronize(model.backend); + + // embed output [feat_dim, S] -> gather per-frame rows + std::vector embed_vec(ggml_nelements(g->output)); + ggml_backend_tensor_get(g->output, embed_vec.data(), 0, embed_vec.size() * sizeof(float)); + const float * embed = embed_vec.data(); + const int64_t F = model.config.feat_dim; + // prepare_avg_tokens_durations + get_tokens_index + const int64_t avg = T / token_count; // num_frames // num_tokens (B = 1) + std::vector durations(static_cast(token_count), avg); + durations.push_back(T - avg * token_count); + std::vector tokens_index(static_cast(T)); + { + int64_t cur = 0; + for (int64_t i = 0; i < S; ++i) { + const int64_t d = durations[static_cast(i)]; + for (int64_t k = 0; k < d; ++k) { + if (cur + k < T) tokens_index[static_cast(cur + k)] = static_cast(i); + } + cur += d; + } + } + TextConditionResult result; + result.T = T; + result.condition.resize(static_cast(T * F)); + for (int64_t i = 0; i < T; ++i) { + const int32_t tok = tokens_index[static_cast(i)]; + std::memcpy(result.condition.data() + i * F, + embed + static_cast(tok) * F, F * sizeof(float)); + } + return result; +} + +// Euler solver with t_shift; mirrors DiffusionModel/EulerSolver exactly. +std::vector run_sampler( + LoadedModel & model, + const ZipVoiceComputeDevice & device, + const TextConditionResult & text, + const std::vector & speech_condition, // [T, F] padded prompt (zeros elsewhere) + int64_t valid_frames, + const std::vector & x0, + int num_step, + float guidance_scale, + float t_shift) { + const auto & config = model.config; + const int64_t T = text.T; + const int64_t F = config.feat_dim; + const bool distill = config.guidance_scale_embed; + if (num_step <= 0 || num_step > 64 || !std::isfinite(t_shift) || t_shift <= 0 || + !std::isfinite(guidance_scale) || guidance_scale < 0 || + x0.size() != static_cast(T * F)) { + throw std::invalid_argument("zipvoice: invalid sampler parameters or noise shape"); + } + + // timesteps + std::vector ts(num_step + 1); + for (int i = 0; i <= num_step; ++i) { + ts[static_cast(i)] = static_cast(i) / num_step; + } + for (auto & v : ts) { + v = t_shift * v / (1.0F + (t_shift - 1.0F) * v); + } + + std::vector x = x0; + if (distill) { + const auto guidance_embedding = timestep_embedding(guidance_scale, config.time_embed_dim); + for (int step = 0; step < num_step; ++step) { + const float t = ts[static_cast(step)]; + const float dt = ts[static_cast(step + 1)] - t; + auto v = run_velocity(model, device, T, 1, x.data(), text.condition.data(), + speech_condition.data(), valid_frames, t, + guidance_embedding.data()); + for (size_t i = 0; i < x.size(); ++i) { + x[i] += v[i] * dt; + } + } + return x; + } + + // base model: batched CFG + const bool use_cfg = guidance_scale != 0.0F; + if (!use_cfg) { + for (int step = 0; step < num_step; ++step) { + const float t = ts[static_cast(step)]; + const float dt = ts[static_cast(step + 1)] - t; + auto v = run_velocity(model, device, T, 1, x.data(), text.condition.data(), + speech_condition.data(), valid_frames, t, nullptr); + for (size_t i = 0; i < x.size(); ++i) x[i] += v[i] * dt; + } + return x; + } + + float effective = guidance_scale; + std::vector xt2, text2, speech2; + for (int step = 0; step < num_step; ++step) { + const float t = ts[static_cast(step)]; + const float dt = ts[static_cast(step + 1)] - t; + float scale = effective; + xt2.assign(x.data(), x.data() + x.size()); + xt2.insert(xt2.end(), x.begin(), x.end()); + text2.assign(text.condition.size(), 0.0F); + text2.insert(text2.end(), text.condition.begin(), text.condition.end()); + if (t > 0.5F) { + speech2.assign(speech_condition.size(), 0.0F); + speech2.insert(speech2.end(), speech_condition.begin(), speech_condition.end()); + } else { + scale = effective * 2.0F; + speech2.assign(speech_condition.begin(), speech_condition.end()); + speech2.insert(speech2.end(), speech_condition.begin(), speech_condition.end()); + } + auto v = run_velocity(model, device, T, 2, xt2.data(), text2.data(), + speech2.data(), valid_frames, t, nullptr); + std::vector combined(x.size()); + for (size_t i = 0; i < x.size(); ++i) { + const float cond = v[x.size() + i]; + const float uncond = v[i]; + combined[i] = (1.0F + scale) * cond - scale * uncond; + } + for (size_t i = 0; i < x.size(); ++i) x[i] += combined[i] * dt; + } + return x; +} + +struct Rng { + uint64_t state; + explicit Rng(uint64_t seed) : state(seed ? seed : 0x9E3779B97F4A7C15ULL) {} + uint64_t next_u64() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; + } + float next_f32() { + return static_cast(static_cast(next_u64() >> 11) / 9007199254740992.0); + } + float normal() { + const float u1 = std::max(next_f32(), 1e-7F); + const float u2 = next_f32(); + return std::sqrt(-2.0F * std::log(u1)) * std::cos(2.0F * static_cast(kPi) * u2); + } +}; + +// tokens.txt vocab: token -> id (tab separated) +std::unordered_map load_tokens(const std::filesystem::path & tokens_path) { + std::unordered_map map; + const auto & path = tokens_path; + std::ifstream f(path); + if (!f) throw std::runtime_error("zipvoice: cannot open " + path.string()); + std::string line; + while (std::getline(f, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + map.emplace(line.substr(0, tab), std::stoi(line.substr(tab + 1))); + } + if (map.empty()) throw std::runtime_error("zipvoice: empty tokens.txt"); + return map; +} + +std::vector utf8_chars(const std::string & s) { + std::vector out; + for (size_t i = 0; i < s.size();) { + size_t len = 1; + const auto c = static_cast(s[i]); + if (c >= 0xF0) len = 4; + else if (c >= 0xE0) len = 3; + else if (c >= 0xC0) len = 2; + out.emplace_back(s, i, len); + i += len; + } + return out; +} + +} // namespace + +void zipvoice_clear_runtime(const ZipVoiceComputeDevice & device) { + if (!device.runtime) return; + std::lock_guard lock(device.runtime->mutex); + device.runtime->model.reset(); + device.runtime->key.clear(); +} + +std::vector zipvoice_vocos_decode_on_device( + const std::string & model_path, + const std::string & vocos_path, + const std::vector & mel_rows, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + std::lock_guard lock(model.mutex); + return decode_vocos(model, vocos_path, mel_rows, compute_threads(device)); +} + +std::vector zipvoice_text_condition( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + int64_t prompt_features_len, + float speed, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + auto result = compute_text_condition( + model, device, tokens, prompt_tokens, prompt_features_len, speed); + return result.condition; +} + +std::vector zipvoice_text_encoder_raw( + const std::string & model_path, + const std::vector & token_ids, + ZipVoiceLayerTaps * layer_taps, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + std::vector cat(token_ids); + cat.push_back(model.config.pad_id); + const int64_t S = static_cast(cat.size()); + + std::lock_guard lock(model.mutex); + TextEncoderGraph * g = nullptr; + if (const auto * slot = model.text_graphs.find(S)) { + g = slot->get(); + } else { + model.text_graphs.clear(); + auto built = build_text_encoder_graph(model.weights, model.config, S, false, model.backend); + model.text_graphs.put(S, std::make_unique(std::move(built))); + g = model.text_graphs.find(S)->get(); + } + upload_leaf(model.backend, g->token_ids, cat.data(), cat.size() * sizeof(int32_t)); + std::vector zeros(static_cast(S), 0.0F); + std::vector ones(static_cast(S), 1.0F); + zeros.back() = -1000.0F; + ones.back() = 0.0F; + upload_leaf(model.backend, g->pad_bias[0], zeros.data(), zeros.size() * sizeof(float)); + upload_leaf(model.backend, g->conv_gate[0], ones.data(), ones.size() * sizeof(float)); + + core::set_backend_threads(model.backend, compute_threads(device)); + const auto status = core::compute_backend_graph(model.backend, g->graph, nullptr, "zipvoice_text"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("zipvoice: text encoder graph compute failed"); + } + ggml_backend_synchronize(model.backend); + + // output [feat_dim, S] -> [S, feat_dim] rows + std::vector out_vec(ggml_nelements(g->output)); + ggml_backend_tensor_get(g->output, out_vec.data(), 0, out_vec.size() * sizeof(float)); + const int64_t F = model.config.feat_dim; + std::vector rows(static_cast(S * F)); + for (int64_t i = 0; i < S; ++i) { + for (int64_t f = 0; f < F; ++f) { + rows[static_cast(i * F + f)] = + out_vec[static_cast(f + i * F)]; + } + } + if (layer_taps != nullptr) { + layer_taps->layers.clear(); + layer_taps->stages.clear(); + for (ggml_tensor * tap : g->stage_taps) { + // raw flat dump in ggml ne order; python/test side knows each + // tap's shape and reorders + std::vector tap_vec(ggml_nelements(tap)); + ggml_backend_tensor_get(tap, tap_vec.data(), 0, tap_vec.size() * sizeof(float)); + layer_taps->stages.push_back(std::move(tap_vec)); + } + for (ggml_tensor * tap : g->layer_taps) { + const int64_t C = tap->ne[0]; + std::vector tap_vec(ggml_nelements(tap)); + ggml_backend_tensor_get(tap, tap_vec.data(), 0, tap_vec.size() * sizeof(float)); + std::vector tap_rows(static_cast(S * C)); + for (int64_t i = 0; i < S; ++i) { + for (int64_t c = 0; c < C; ++c) { + tap_rows[static_cast(i * C + c)] = + tap_vec[static_cast(c + i * C)]; + } + } + layer_taps->layers.push_back(std::move(tap_rows)); + } + } + return rows; +} + +std::vector zipvoice_velocity( + const std::string & model_path, + const std::vector & xt, + const std::vector & text_condition, + const std::vector & speech_condition, + int64_t features_len, + float t, + float guidance_scale, + const ZipVoiceComputeDevice & device, + int batch_size, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + const int64_t T = features_len; + const float * guidance = nullptr; + std::vector guidance_embedding; + if (model.config.guidance_scale_embed) { + guidance_embedding = timestep_embedding(guidance_scale, model.config.time_embed_dim); + guidance = guidance_embedding.data(); + } + const size_t expected = static_cast(T * model.config.feat_dim * batch_size); + if (T <= 0 || batch_size <= 0 || xt.size() != expected || + text_condition.size() != expected || speech_condition.size() != expected) { + throw std::invalid_argument("zipvoice: velocity input shape mismatch"); + } + return run_velocity(model, device, T, batch_size, xt.data(), text_condition.data(), + speech_condition.data(), features_len, t, guidance); +} + +std::vector zipvoice_sample( + const std::string & model_path, + const std::vector & tokens, + const std::vector & prompt_tokens, + const std::vector & prompt_features, + int64_t prompt_features_len, + const std::vector & x0, + int num_steps, + float guidance_scale, + float t_shift, + float speed, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + auto owner = load_model(model_path, SidecarResolver{resources}, device); + auto & model = *owner; + auto text = compute_text_condition( + model, device, tokens, prompt_tokens, prompt_features_len, speed); + const int64_t T = text.T; + const int64_t F = model.config.feat_dim; + if (prompt_features_len <= 0 || prompt_features.size() < static_cast(prompt_features_len * F)) { + throw std::invalid_argument("zipvoice: prompt feature shape mismatch"); + } + std::vector speech(static_cast(T * F), 0.0F); + for (int64_t i = 0; i < prompt_features_len && i < T; ++i) { + std::memcpy(speech.data() + i * F, + prompt_features.data() + static_cast(i * F), F * sizeof(float)); + } + return run_sampler(model, device, text, speech, T, x0, num_steps, guidance_scale, t_shift); +} + +ZipVoiceSynthesisResult zipvoice_synthesize( + const std::string & model_path, + const std::string & vocos_path, + const ZipVoiceSynthesisRequest & request, + const ZipVoiceComputeDevice & device, + const engine::assets::ResourceBundle * resources) { + if (request.ref_audio.empty() || request.ref_sample_rate <= 0 || + !std::isfinite(request.feat_scale) || request.feat_scale <= 0) { + throw std::invalid_argument("zipvoice: valid reference audio and positive feature scale are required"); + } + const SidecarResolver sidecars{resources}; + auto owner = load_model(model_path, sidecars, device); + auto & model = *owner; + + // tokens + std::vector tokens = request.token_ids; + std::vector prompt_tokens = request.prompt_token_ids; + if (tokens.empty() || prompt_tokens.empty()) { + // Keep the dictionary frontend with the session, just like the graphs. + // Loading its tables and Jieba dictionary for every chunk is avoidable. + // Serialize encoding too: Emilia lazily initializes its phonemizer. + std::lock_guard lock(model.mutex); + const std::filesystem::path path(model_path); + const std::filesystem::path dir = + std::filesystem::is_directory(path) ? path : path.parent_path(); + const auto vocab = load_tokens(sidecars.path("tokens", dir, "tokens.txt")); + std::unique_ptr phonemizer; + EmiliaTokenizer * emilia = nullptr; + if (request.tokenizer == "espeak") { + phonemizer = std::make_unique( + std::filesystem::path{request.espeak_library_path}, request.espeak_data_path, + std::vector{request.lang}); + } else if (request.tokenizer == "emilia") { + // Chinese/mixed frontend: needs the baked pypinyin tables (from + // the bundle or next to tokens.txt) and an espeak-ng installation + // for English segments. + EmiliaTokenizer::TablePaths tables; + tables.chars = sidecars.path("zh_chars", dir, "zh_chars.tsv"); + tables.phrases = sidecars.path("zh_phrases", dir, "zh_phrases.tsv"); + tables.syllables = sidecars.path("zh_syllables", dir, "zh_syllables.tsv"); + tables.jieba_dict = sidecars.path("zh_jieba_dict", dir, "zh_jieba_dict.txt"); + tables.hmm_model = sidecars.path("zh_hmm_model", dir, "zh_hmm_model.txt"); + const std::vector key{ + sidecars.path("tokens", dir, "tokens.txt").string(), + tables.chars.string(), tables.phrases.string(), tables.syllables.string(), + tables.jieba_dict.string(), tables.hmm_model.string(), + request.espeak_library_path, request.espeak_data_path, request.lang}; + if (!model.tokenizers.find(key)) { + model.tokenizers.clear(); + model.tokenizers.put(key, std::make_unique( + tables, vocab, EmiliaTokenizer::EspeakConfig{ + request.espeak_library_path, request.espeak_data_path, request.lang})); + } + emilia = model.tokenizers.find(key)->get(); + } else if (request.tokenizer != "simple") { + throw std::invalid_argument("zipvoice: tokenizer must be espeak, emilia, or simple"); + } + const auto encode = [&](const std::string & text) { + if (emilia) { + return emilia->encode(text); + } + std::string phones = text; + if (phonemizer) { + phones.clear(); + size_t start = 0; + for (size_t i = 0; i <= text.size(); ++i) { + if (i != text.size() && std::string(";:,.!?\"").find(text[i]) == std::string::npos) continue; + const auto segment = text.substr(start, i - start); + if (!segment.empty()) { + if (!phones.empty() && std::isspace(static_cast(segment.front()))) phones += ' '; + phones += phonemizer->phonemize(segment, 2); + } + if (i != text.size()) phones += text[i]; + start = i + 1; + } + } + std::vector ids; + for (const auto & ch : utf8_chars(phones)) { + if (const auto it = vocab.find(ch); it != vocab.end()) { + ids.push_back(it->second); + } + } + return ids; + }; + if (tokens.empty()) tokens = encode(request.text); + if (prompt_tokens.empty()) prompt_tokens = encode(request.ref_text); + } + + // prompt audio -> 24k mono + auto wav = audio::convert_interleaved_audio_to_mono_torchaudio_sinc_hann_resampled( + request.ref_audio, request.ref_sample_rate, request.ref_channels, kSampleRate); + float prompt_rms = 0.0F; + for (const float s : wav) prompt_rms += s * s; + prompt_rms = std::sqrt(prompt_rms / std::max(1, wav.size())); + if (request.target_rms > 0.0F && prompt_rms < request.target_rms) { + const float gain = request.target_rms / std::max(prompt_rms, 1e-8F); + for (auto & s : wav) s *= gain; + } + + const auto start = std::chrono::steady_clock::now(); + auto mel = zipvoice_logmel(wav); // [T_p, 100] + const int64_t prompt_features_len = static_cast(mel.size()) / kNMel; + const float feat_scale = request.feat_scale; + for (auto & v : mel) v *= feat_scale; + + auto text = compute_text_condition( + model, device, tokens, prompt_tokens, prompt_features_len, request.speed); + const int64_t T = text.T; + const int64_t F = model.config.feat_dim; + + std::vector speech(static_cast(T * F), 0.0F); + std::memcpy(speech.data(), mel.data(), + static_cast(prompt_features_len) * static_cast(F) * sizeof(float)); + + Rng rng(request.seed); + std::vector x0(static_cast(T * F)); + for (auto & v : x0) v = rng.normal(); + + auto x1 = run_sampler( + model, device, text, speech, T, x0, request.num_steps, + request.guidance_scale, request.t_shift); + + // strip prompt frames, unscale features + const int64_t out_frames = T - prompt_features_len; + std::vector features(static_cast(out_frames * F)); + for (int64_t i = 0; i < out_frames; ++i) { + for (int64_t f = 0; f < F; ++f) { + features[static_cast(i * F + f)] = + x1[static_cast((prompt_features_len + i) * F + f)] / feat_scale; + } + } + + std::vector audio; + { + std::lock_guard lock(model.mutex); + audio = decode_vocos(model, vocos_path, features, compute_threads(device)); + } + for (auto & s : audio) s = std::clamp(s, -1.0F, 1.0F); + if (request.target_rms > 0.0F && prompt_rms < request.target_rms && prompt_rms > 0.0F) { + const float gain = prompt_rms / request.target_rms; + for (auto & s : audio) s *= gain; + } + + ZipVoiceSynthesisResult result; + result.sample_rate = kSampleRate; + result.audio = std::move(audio); + const auto end = std::chrono::steady_clock::now(); + result.model_seconds = std::chrono::duration(end - start).count(); + result.audio_seconds = static_cast(result.audio.size()) / kSampleRate; + return result; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/weights.cpp b/src/community_models/zipvoice/weights.cpp new file mode 100644 index 000000000..1104c018e --- /dev/null +++ b/src/community_models/zipvoice/weights.cpp @@ -0,0 +1,255 @@ +#include "engine/community_models/zipvoice/weights.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/io/json.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +namespace { + +std::string read_file(const std::filesystem::path & path) { + std::ifstream stream(path, std::ios::binary); + if (!stream) { + throw std::runtime_error("cannot open " + path.string()); + } + std::ostringstream buffer; + buffer << stream.rdbuf(); + return buffer.str(); +} + +std::vector require_int_list(const io::json::Value & object, const std::string & key) { + const auto & array = object.require(key); + std::vector values; + for (const auto & item : array.as_array()) { + values.push_back(static_cast(item.as_number())); + } + return values; +} + +} // namespace + +ZipVoiceConfig load_zipvoice_config( + const std::filesystem::path & model_dir, + const engine::assets::TensorSource * probe, + const std::filesystem::path * config_path) { + ZipVoiceConfig config; + const auto config_path_resolved = + config_path != nullptr ? *config_path : model_dir / "model.json"; + if (std::filesystem::is_regular_file(config_path_resolved)) { + const auto root = io::json::parse_file(config_path_resolved); + const auto & model = root.require("model"); + config.fm_downsampling_factor = require_int_list(model, "fm_decoder_downsampling_factor"); + config.fm_num_layers = require_int_list(model, "fm_decoder_num_layers"); + config.fm_cnn_kernel = require_int_list(model, "fm_decoder_cnn_module_kernel"); + config.fm_feedforward_dim = io::json::require_i32(model, "fm_decoder_feedforward_dim"); + config.fm_num_heads = io::json::require_i32(model, "fm_decoder_num_heads"); + config.fm_dim = io::json::require_i32(model, "fm_decoder_dim"); + config.text_num_layers = io::json::require_i32(model, "text_encoder_num_layers"); + config.text_feedforward_dim = io::json::require_i32(model, "text_encoder_feedforward_dim"); + config.text_cnn_kernel = io::json::require_i32(model, "text_encoder_cnn_module_kernel"); + config.text_num_heads = io::json::require_i32(model, "text_encoder_num_heads"); + config.text_dim = io::json::require_i32(model, "text_encoder_dim"); + config.query_head_dim = io::json::require_i32(model, "query_head_dim"); + config.value_head_dim = io::json::require_i32(model, "value_head_dim"); + config.pos_head_dim = io::json::require_i32(model, "pos_head_dim"); + config.pos_dim = io::json::require_i32(model, "pos_dim"); + config.time_embed_dim = io::json::require_i32(model, "time_embed_dim"); + config.text_embed_dim = io::json::require_i32(model, "text_embed_dim"); + config.feat_dim = io::json::require_i32(model, "feat_dim"); + const auto * feature = root.find("feature"); + if (feature != nullptr) { + config.sampling_rate = io::json::require_i32(*feature, "sampling_rate"); + } + } else { + // Defaults matching the released zipvoice / zipvoice_distill config. + } + if (config.fm_downsampling_factor.size() != config.fm_num_layers.size() || + config.fm_downsampling_factor.size() != config.fm_cnn_kernel.size()) { + throw std::runtime_error("zipvoice: stack config length mismatch in model.json"); + } + if (probe != nullptr) { + config.guidance_scale_embed = probe->has_tensor("fm_decoder.guidance_scale_embed.weight"); + } + return config; +} + +ZipVoiceWeights load_zipvoice_weights( + const engine::assets::TensorSource & raw_source, + const std::string & prefix, + const ZipVoiceConfig & config, + ggml_backend_t backend, + core::BackendType backend_type) { + std::shared_ptr source = + engine::assets::make_prefixed_tensor_source( + std::shared_ptr( + std::shared_ptr(), &raw_source), + prefix); + + ZipVoiceWeights weights; + weights.store = std::make_shared( + backend, backend_type, "zipvoice.weights", 2ULL * 1024ULL * 1024ULL * 1024ULL); + + // Dimension arguments are PHYSICAL ggml order (ne0 fastest). -1 accepts + // any value; non-wildcard entries are validated against the checkpoint + // metadata (which stores torch order, reversed here). + const auto t2 = [&](const std::string & name, int64_t d0, int64_t d1) { + const auto meta = source->require_metadata(name); + if (meta.shape.size() != 2) { + throw std::runtime_error("zipvoice: " + name + " must be rank 2"); + } + if ((d0 >= 0 && d0 != meta.shape[1]) || (d1 >= 0 && d1 != meta.shape[0])) { + throw std::runtime_error("zipvoice: " + name + " shape mismatch"); + } + return weights.store->load_f32_tensor(*source, name, meta.shape); + }; + const auto t1 = [&](const std::string & name, int64_t d0) { + const auto meta = source->require_metadata(name); + if (meta.shape.size() != 1) { + throw std::runtime_error("zipvoice: " + name + " must be rank 1"); + } + if (d0 >= 0 && d0 != meta.shape[0]) { + throw std::runtime_error("zipvoice: " + name + " shape mismatch"); + } + return weights.store->load_f32_tensor(*source, name, meta.shape); + }; + const auto t3 = [&](const std::string & name, int64_t d0, int64_t d1, int64_t d2) { + const auto meta = source->require_metadata(name); + if (meta.shape.size() != 3) { + throw std::runtime_error("zipvoice: " + name + " must be rank 3"); + } + if ((d0 >= 0 && d0 != meta.shape[2]) || (d1 >= 0 && d1 != meta.shape[1]) || + (d2 >= 0 && d2 != meta.shape[0])) { + throw std::runtime_error("zipvoice: " + name + " shape mismatch"); + } + return weights.store->load_f32_tensor(*source, name, meta.shape); + }; + + const auto load_zipformer = [&](TTSZipformerWeights & w, const std::string & base, + int encoder_dim, int feedforward_dim, + const std::vector & downsampling, + const std::vector & num_layers, + const std::vector & kernels, + int num_heads, bool with_time) { + w.in_proj_w = t2(base + ".in_proj.weight", -1, encoder_dim); + w.in_proj_b = t1(base + ".in_proj.bias", encoder_dim); + w.out_proj_w = t2(base + ".out_proj.weight", -1, -1); + w.out_proj_b = t1(base + ".out_proj.bias", -1); + if (with_time) { + w.time_mlp0_w = t2(base + ".time_embed.0.weight", config.time_embed_dim, config.time_embed_dim * 2); + w.time_mlp0_b = t1(base + ".time_embed.0.bias", config.time_embed_dim * 2); + w.time_mlp2_w = t2(base + ".time_embed.2.weight", config.time_embed_dim * 2, config.time_embed_dim); + w.time_mlp2_b = t1(base + ".time_embed.2.bias", config.time_embed_dim); + if (config.guidance_scale_embed) { + w.guidance_embed_w = t2(base + ".guidance_scale_embed.weight", config.time_embed_dim, config.time_embed_dim); + } + } + const size_t num_stacks = downsampling.size(); + w.stacks.resize(num_stacks); + for (size_t s = 0; s < num_stacks; ++s) { + auto & stack = w.stacks[s]; + const std::string sp = base + ".encoders." + std::to_string(s); + stack.layers.resize(static_cast(num_layers[s])); + for (size_t l = 0; l < stack.layers.size(); ++l) { + auto & layer = stack.layers[l]; + const std::string lp = sp + (downsampling[s] > 1 ? ".encoder" : "") + + ".layers." + std::to_string(l); + layer.bypass_scale = t1(lp + ".bypass.bypass_scale", encoder_dim); + layer.bypass_mid_scale = t1(lp + ".bypass_mid.bypass_scale", encoder_dim); + layer.attn_in_proj_w = t2(lp + ".self_attn_weights.in_proj.weight", encoder_dim, -1); + layer.attn_in_proj_b = t1(lp + ".self_attn_weights.in_proj.bias", -1); + layer.linear_pos_w = t2(lp + ".self_attn_weights.linear_pos.weight", + config.pos_dim, num_heads * config.pos_head_dim); + layer.sa1_in_w = t2(lp + ".self_attn1.in_proj.weight", encoder_dim, num_heads * config.value_head_dim); + layer.sa1_in_b = t1(lp + ".self_attn1.in_proj.bias", num_heads * config.value_head_dim); + layer.sa1_out_w = t2(lp + ".self_attn1.out_proj.weight", num_heads * config.value_head_dim, encoder_dim); + layer.sa1_out_b = t1(lp + ".self_attn1.out_proj.bias", encoder_dim); + layer.sa2_in_w = t2(lp + ".self_attn2.in_proj.weight", encoder_dim, num_heads * config.value_head_dim); + layer.sa2_in_b = t1(lp + ".self_attn2.in_proj.bias", num_heads * config.value_head_dim); + layer.sa2_out_w = t2(lp + ".self_attn2.out_proj.weight", num_heads * config.value_head_dim, encoder_dim); + layer.sa2_out_b = t1(lp + ".self_attn2.out_proj.bias", encoder_dim); + const int ff1_hidden = feedforward_dim * 3 / 4; + const int ff3_hidden = feedforward_dim * 5 / 4; + layer.ff1_in_w = t2(lp + ".feed_forward1.in_proj.weight", encoder_dim, ff1_hidden); + layer.ff1_in_b = t1(lp + ".feed_forward1.in_proj.bias", ff1_hidden); + layer.ff1_out_w = t2(lp + ".feed_forward1.out_proj.weight", ff1_hidden, encoder_dim); + layer.ff1_out_b = t1(lp + ".feed_forward1.out_proj.bias", encoder_dim); + layer.ff2_in_w = t2(lp + ".feed_forward2.in_proj.weight", encoder_dim, feedforward_dim); + layer.ff2_in_b = t1(lp + ".feed_forward2.in_proj.bias", feedforward_dim); + layer.ff2_out_w = t2(lp + ".feed_forward2.out_proj.weight", feedforward_dim, encoder_dim); + layer.ff2_out_b = t1(lp + ".feed_forward2.out_proj.bias", encoder_dim); + layer.ff3_in_w = t2(lp + ".feed_forward3.in_proj.weight", encoder_dim, ff3_hidden); + layer.ff3_in_b = t1(lp + ".feed_forward3.in_proj.bias", ff3_hidden); + layer.ff3_out_w = t2(lp + ".feed_forward3.out_proj.weight", ff3_hidden, encoder_dim); + layer.ff3_out_b = t1(lp + ".feed_forward3.out_proj.bias", encoder_dim); + const int na_hidden = encoder_dim * 3 / 4; + layer.na_in_w = t2(lp + ".nonlin_attention.in_proj.weight", encoder_dim, na_hidden * 3); + layer.na_in_b = t1(lp + ".nonlin_attention.in_proj.bias", na_hidden * 3); + layer.na_out_w = t2(lp + ".nonlin_attention.out_proj.weight", na_hidden, encoder_dim); + layer.na_out_b = t1(lp + ".nonlin_attention.out_proj.bias", encoder_dim); + const int kernel = kernels[s]; + for (int m = 1; m <= 2; ++m) { + const std::string mp = lp + ".conv_module" + std::to_string(m); + core::TensorValue &in_w = m == 1 ? layer.cm1_in_w : layer.cm2_in_w; + core::TensorValue &in_b = m == 1 ? layer.cm1_in_b : layer.cm2_in_b; + core::TensorValue &cv_w = m == 1 ? layer.cm1_conv_w : layer.cm2_conv_w; + core::TensorValue &cv_b = m == 1 ? layer.cm1_conv_b : layer.cm2_conv_b; + core::TensorValue &out_w = m == 1 ? layer.cm1_out_w : layer.cm2_out_w; + core::TensorValue &out_b = m == 1 ? layer.cm1_out_b : layer.cm2_out_b; + in_w = t2(mp + ".in_proj.weight", encoder_dim, encoder_dim * 2); + in_b = t1(mp + ".in_proj.bias", encoder_dim * 2); + cv_w = t3(mp + ".depthwise_conv.weight", kernel, 1, encoder_dim); + cv_b = t1(mp + ".depthwise_conv.bias", encoder_dim); + out_w = t2(mp + ".out_proj.weight", encoder_dim, encoder_dim); + out_b = t1(mp + ".out_proj.bias", encoder_dim); + } + layer.norm_bias = t1(lp + ".norm.bias", encoder_dim); + const auto log_scale = source->require_f32(lp + ".norm.log_scale"); + if (log_scale.size() != 1) { + throw std::runtime_error("zipvoice: " + lp + ".norm.log_scale must be a scalar"); + } + layer.norm_log_scale = log_scale[0]; + } + if (with_time) { + stack.time_proj_w = t2(sp + (downsampling[s] > 1 ? ".encoder" : "") + ".time_emb.1.weight", + config.time_embed_dim, encoder_dim); + stack.time_proj_b = t1(sp + (downsampling[s] > 1 ? ".encoder" : "") + ".time_emb.1.bias", encoder_dim); + } + if (downsampling[s] > 1) { + const auto bias = source->require_f32( + sp + ".downsample.bias", + std::vector{downsampling[s]}); + stack.downsample_bias = bias; + const float maximum = *std::max_element(bias.begin(), bias.end()); + float sum = 0.0F; + for (float & value : stack.downsample_bias) { + value = std::exp(value - maximum); + sum += value; + } + for (float & value : stack.downsample_bias) value /= sum; + stack.out_combiner_scale = t1(sp + ".out_combiner.bypass_scale", encoder_dim); + } + } + }; + + load_zipformer(weights.fm_decoder, "fm_decoder", config.fm_dim, config.fm_feedforward_dim, + config.fm_downsampling_factor, config.fm_num_layers, config.fm_cnn_kernel, + config.fm_num_heads, true); + load_zipformer(weights.text_encoder, "text_encoder", config.text_dim, config.text_feedforward_dim, + {1}, {config.text_num_layers}, {config.text_cnn_kernel}, + config.text_num_heads, false); + weights.text_encoder.embed_w = t2("embed.weight", config.text_embed_dim, config.vocab_size); + + weights.store->upload(); + source->release_storage(); + return weights; +} + +} // namespace engine::models::zipvoice diff --git a/src/community_models/zipvoice/zipformer.cpp b/src/community_models/zipvoice/zipformer.cpp new file mode 100644 index 000000000..e2c24ac11 --- /dev/null +++ b/src/community_models/zipvoice/zipformer.cpp @@ -0,0 +1,931 @@ +// TTSZipformer graph construction for the ZipVoice community port. +// +// Reference: zipvoice/models/modules/zipformer.py (k2-fsa/ZipVoice, +// Apache-2.0) evaluated in inference mode. Balancer / Whiten / Identity / +// dropout modules are no-ops; BypassModule reduces to a per-channel learned +// residual scale; ScaledLinear checkpoints already contain folded scales. +// +// Physical layouts (ne0 fastest): +// activations: [C (ne0), T (ne1), B (ne2)] +// attention: [src (ne0), tgt (ne1), H (ne2), B (ne3)] +// Softmax always runs over ne0 (keys), matching torch softmax(dim=-1) on +// (H, B, tgt, src). +// +// The compact relative-position attention lowers torch's overlapping +// as_strided view into an explicit strided ggml view over the contiguous +// relative scores R [2T-1 (ne0), T (ne1), H, B]: +// abs[s, t] = R_flat[(T - 1) + t * (2T - 2) + s] +// where R_flat[m + t * (2T - 1)] = p_t . w_m. Validated against the +// moonlight fixture (tests/zipvoice). + +#include "engine/community_models/zipvoice/zipformer.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/lookup_modules.h" + +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::zipvoice { +ZipVoiceGraphResources::~ZipVoiceGraphResources() { + if (backend && graph) core::release_backend_graph_resources(backend, graph, true); + if (gallocr) ggml_gallocr_free(static_cast(gallocr)); + if (buffer) ggml_backend_buffer_free(buffer); + if (ctx) ggml_free(ctx); + if (tensor_ctx) ggml_free(tensor_ctx); +} + +namespace { + +constexpr float kPadBias = -1000.0F; + +// --------------------------------------------------------------------------- +// small helpers (raw ggml, channel-fastest layout) + +// Physical views remain explicit for Zipformer's overlapping relative shift; +// ordinary projections, activation, normalization and convolution use modules. +struct GraphBuilder : core::ModuleBuildContext { + GraphBuilder(ggml_context * c, core::BackendType type) { ggml = c; backend_type = type; } + operator ggml_context *() const { return ggml; } +}; +core::TensorValue value(ggml_tensor * t) { + return core::wrap_tensor(t, core::TensorShape::from_dims({t->ne[3], t->ne[2], t->ne[1], t->ne[0]}), t->type); +} +ggml_tensor * t_cont(GraphBuilder & ctx, ggml_tensor * t) { + return core::ensure_backend_addressable_layout(ctx, value(t)).tensor; +} +ggml_tensor * t_permute(GraphBuilder & ctx, ggml_tensor * t, int p0, int p1, int p2, int p3) { + const int physical[4] = {p0, p1, p2, p3}; + std::array axes; + for (int source = 0; source < 4; ++source) axes[3 - physical[source]] = 3 - source; + return modules::TransposeModule({axes, 4}).build(ctx, value(t)).tensor; +} +ggml_tensor * t_reshape_2d(GraphBuilder & ctx, ggml_tensor * t, int64_t n0, int64_t n1) { + return core::reshape_tensor(ctx, value(t), core::TensorShape::from_dims({n1, n0})).tensor; +} +ggml_tensor * t_reshape_3d(GraphBuilder & ctx, ggml_tensor * t, int64_t n0, int64_t n1, int64_t n2) { + return core::reshape_tensor(ctx, value(t), core::TensorShape::from_dims({n2, n1, n0})).tensor; +} +ggml_tensor * t_reshape_4d(GraphBuilder & ctx, ggml_tensor * t, int64_t n0, int64_t n1, int64_t n2, int64_t n3) { + return core::reshape_tensor(ctx, value(t), core::TensorShape::from_dims({n3, n2, n1, n0})).tensor; +} +ggml_tensor * t_rows(GraphBuilder & ctx, ggml_tensor * table, ggml_tensor * indices) { + return modules::EmbeddingModule({table->ne[1], table->ne[0]}).build(ctx, + core::wrap_tensor(indices, core::TensorShape::from_dims({indices->ne[0]}), GGML_TYPE_I32), + core::wrap_tensor(table, core::TensorShape::from_dims({table->ne[1], table->ne[0]}), table->type)).tensor; +} +ggml_tensor * t_matmul(GraphBuilder & ctx, ggml_tensor * a, ggml_tensor * b) { + if (a->ne[2] != b->ne[2] || a->ne[3] != b->ne[3]) { + // MatMulModule requires equal batch dimensions. Relative-position + // attention and weighted downsampling rely on ggml's batch broadcast. + return ggml_mul_mat(ctx, t_cont(ctx, a), t_cont(ctx, b)); + } + const auto rhs = modules::TransposeModule({{0, 1, 3, 2}, 4}).build(ctx, value(a)); + return modules::MatMulModule().build(ctx, value(b), rhs).tensor; +} +ggml_tensor * t_add(GraphBuilder & ctx, ggml_tensor * a, ggml_tensor * b) { + auto rhs = modules::RepeatModule({value(a).shape}).build(ctx, value(b)); + return modules::AddModule().build(ctx, value(a), rhs).tensor; +} +ggml_tensor * t_mul(GraphBuilder & ctx, ggml_tensor * a, ggml_tensor * b) { + auto rhs = modules::RepeatModule({value(a).shape}).build(ctx, value(b)); + return modules::MulModule().build(ctx, value(a), rhs).tensor; +} +ggml_tensor * t_linear(GraphBuilder & ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) { + modules::LinearWeights weights{core::wrap_tensor(w, core::TensorShape::from_dims({w->ne[1], w->ne[0]}), w->type), std::nullopt}; + if (b) weights.bias = core::wrap_tensor(b, core::TensorShape::from_dims({b->ne[0]}), b->type); + return modules::LinearModule({w->ne[0], w->ne[1], b != nullptr}).build(ctx, value(x), weights).tensor; +} +ggml_tensor * swoosh_l(GraphBuilder & ctx, ggml_tensor * x) { + return modules::SwooshLModule().build(ctx, value(x)).tensor; +} +ggml_tensor * swoosh_r(GraphBuilder & ctx, ggml_tensor * x) { + return modules::SwooshRModule().build(ctx, value(x)).tensor; +} +ggml_tensor * bias_norm(GraphBuilder & ctx, ggml_tensor * x, ggml_tensor * bias, float log_scale) { + return modules::BiasNormModule({x->ne[0]}).build(ctx, value(x), { + core::wrap_tensor(bias, core::TensorShape::from_dims({bias->ne[0]})), log_scale}).tensor; +} + +std::vector compact_relative_position(int64_t seq, int64_t pos_dim) { + constexpr float pi = 3.14159265358979323846F; + const int64_t length = 2 * seq - 1; + std::vector table(static_cast(length * pos_dim), 0.0F); + const float compression = std::sqrt(static_cast(pos_dim)); + const float length_scale = static_cast(pos_dim) / (2.0F * pi); + for (int64_t row = 0; row < length; ++row) { + const float x = static_cast(row - (seq - 1)); + const float sign = x < 0.0F ? -1.0F : (x > 0.0F ? 1.0F : 0.0F); + const float compressed = compression * sign * + (std::log(std::fabs(x) + compression) - std::log(compression)); + const float angle = std::atan(compressed / length_scale); + for (int64_t dim = 0; dim < pos_dim / 2; ++dim) { + const float frequency = static_cast(dim + 1); + table[static_cast(row * pos_dim + 2 * dim)] = std::cos(angle * frequency); + table[static_cast(row * pos_dim + 2 * dim + 1)] = std::sin(angle * frequency); + } + table[static_cast(row * pos_dim + pos_dim - 1)] = 1.0F; + } + return table; +} + +// Host constant factory. On CPU (inline ggml ctx) values are written +// directly; on CUDA (no_alloc ctx) they are staged and uploaded later. +struct StagedConst { + ggml_tensor * tensor; + std::vector bytes; +}; + +struct Consts { + // Tensors are created in the dedicated tensor context (never the graph + // context) so ggml_backend_alloc_ctx_tensors can give them a private + // backend buffer that the graph arena never aliases. + ggml_context * ctx = nullptr; + std::vector * staged = nullptr; + int index = 0; + + template + ggml_tensor * make(const T * data, size_t count, ggml_type type, + std::initializer_list dims) { + std::vector d(dims); + ggml_tensor * t = nullptr; + switch (d.size()) { + case 1: t = ggml_new_tensor_1d(ctx, type, d[0]); break; + case 2: t = ggml_new_tensor_2d(ctx, type, d[0], d[1]); break; + case 3: t = ggml_new_tensor_3d(ctx, type, d[0], d[1], d[2]); break; + case 4: t = ggml_new_tensor_4d(ctx, type, d[0], d[1], d[2], d[3]); break; + default: throw std::runtime_error("zipvoice: bad const rank"); + } + ggml_set_name(t, ("zv_c" + std::to_string(index++)).c_str()); + const size_t bytes = count * sizeof(T); + if (t->data != nullptr) { + std::memcpy(t->data, data, bytes); + } else if (staged != nullptr) { + const auto * raw = reinterpret_cast(data); + staged->push_back({t, std::vector(raw, raw + bytes)}); + } else { + throw std::runtime_error("zipvoice: no home for graph constant"); + } + return t; + } + + ggml_tensor * f32(std::vector values, std::initializer_list dims) { + return make(values.data(), values.size(), GGML_TYPE_F32, dims); + } + ggml_tensor * i32(std::vector values, std::initializer_list dims) { + return make(values.data(), values.size(), GGML_TYPE_I32, dims); + } +}; + +// --------------------------------------------------------------------------- +// attention pieces + +// RelPositionMultiheadAttentionWeights (inference path). +// src [C, T, B] -> weights [src(ne0), tgt(ne1), H(ne2), B(ne3)]. +ggml_tensor * attention_weights( + GraphBuilder & ctx, + Consts & consts, + const ZipLayerWeights & w, + ggml_tensor * src, + ggml_tensor * attn_bias, // [T, 1, 1, 1] additive + int64_t H, + int64_t qh, + int64_t ph, + int64_t pos_dim, + int64_t T, + int64_t B, + ggml_tensor ** tap_qtb = nullptr, + ggml_tensor ** tap_ktb = nullptr, + ggml_tensor ** tap_inproj = nullptr, + ggml_tensor ** tap_scores = nullptr, + ggml_tensor ** tap_pos = nullptr) { + const int64_t q_total = H * qh; + const int64_t M = 2 * T - 1; + + auto * projected = t_linear(ctx, src, w.attn_in_proj_w.tensor, w.attn_in_proj_b.tensor); + if (tap_inproj != nullptr) *tap_inproj = projected; + // [C_total, T, B]: slice channels, reshape (d, H), permute to (d, T, H, B) + // slice [H*per] channels at `offset` (per-head split, head fastest after + // the per-channel dim) -> [per, T, H, B] + const auto qktb = [&](int64_t offset, int64_t total, int64_t per) { + (void)total; + // channel slice as a 4D view [per, H, T, B]: strides for T/B come + // from the SOURCE (frame stride = full channel width), reshape would + // assert on this non-contiguous layout + auto * four = ggml_view_4d(ctx, projected, per, H, T, B, + per * sizeof(float), projected->nb[1], + projected->nb[2], offset * sizeof(float)); + return t_cont(ctx, t_permute(ctx, four, 0, 2, 1, 3)); // [per, T, H, B] + }; + auto * q_tb = qktb(0, q_total, qh); + auto * k_tb = qktb(q_total, q_total, qh); + auto * p_tb = qktb(2 * q_total, H * ph, ph); + if (tap_qtb != nullptr) *tap_qtb = q_tb; + if (tap_ktb != nullptr) *tap_ktb = k_tb; + + // Raw mul_mat is retained for batched contractions: MatMulModule + // requires equal batch dimensions and cannot express the position broadcast. + // scores[src, tgt, H, B] = k^T q + auto * scores = t_matmul(ctx, k_tb, q_tb); + if (tap_scores != nullptr) *tap_scores = scores; + + // relative position scores: R[m, t] = p_t . w_m as [M, T, H, B] + // (mul_mat broadcasts the degenerate batch dims of the pos operand) + auto * pos_emb = consts.f32(compact_relative_position(T, pos_dim), {pos_dim, M}); + auto * pos_proj = t_matmul(ctx, w.linear_pos_w.tensor, pos_emb); // [H*ph, M] + auto * pos_4d = t_reshape_4d(ctx, pos_proj, ph, H, M, 1); + auto * pos_tb = t_cont(ctx, t_permute(ctx, pos_4d, 0, 2, 1, 3)); // [ph, M, H, 1] + auto * rel_t = t_matmul(ctx, pos_tb, p_tb); // [M, T, H, B] contiguous + + // overlapping strided view (torch as_strided lowering): + // abs[s, t] = flat[(T - 1) + t * (2T - 2) + s] + const size_t nb0 = sizeof(float); + const size_t nb1 = static_cast(2 * T - 2) * nb0; + const size_t nb2 = static_cast(M) * static_cast(T) * nb0; + const size_t nb3 = nb2 * static_cast(H); + auto * pos_abs = ggml_view_4d(ctx, rel_t, T, T, H, B, nb1, nb2, nb3, + static_cast(T - 1) * nb0); + auto * pos_final = t_cont(ctx, pos_abs); // [src, tgt, H, B] + + auto * with_pos = t_add(ctx, scores, pos_final); + if (tap_pos != nullptr) *tap_pos = with_pos; + auto * biased = t_add(ctx, with_pos, attn_bias); + return modules::SoftmaxModule().build(ctx, value(biased)).tensor; +} + +// SelfAttention (value path). weights [src, tgt, H, B]; x [C, T, B]. +ggml_tensor * self_attention( + GraphBuilder & ctx, + const ZipLayerWeights & w, + int which, // 1 or 2 + ggml_tensor * x, + ggml_tensor * attn_weights, + int64_t H, + int64_t vh, + int64_t T, + int64_t B) { + const auto & in_w = which == 1 ? w.sa1_in_w : w.sa2_in_w; + const auto & in_b = which == 1 ? w.sa1_in_b : w.sa2_in_b; + const auto & out_w = which == 1 ? w.sa1_out_w : w.sa2_out_w; + const auto & out_b = which == 1 ? w.sa1_out_b : w.sa2_out_b; + + auto * v = t_linear(ctx, x, in_w.tensor, in_b.tensor); // [H*vh, T, B] + auto * v4 = t_reshape_4d(ctx, v, vh, H, T, B); + // [T(src), vh, H, B] for the contraction over src + auto * v_tb = t_cont(ctx, t_permute(ctx, v4, 1, 2, 0, 3)); + auto * attended = t_matmul(ctx, attn_weights, v_tb); // [tgt, vh, H, B] + // back to channel-fastest [H*vh, T, B] + auto * ch = t_cont(ctx, t_permute(ctx, attended, 2, 0, 1, 3)); // [vh, H, T, B] + auto * ch2 = t_reshape_2d(ctx, ch, H * vh, T * B); + auto * out = t_linear(ctx, ch2, out_w.tensor, out_b.tensor); // [C, T*B] + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// NonlinAttention (uses head-0 weights only). +ggml_tensor * nonlin_attention( + GraphBuilder & ctx, + const ZipLayerWeights & w, + ggml_tensor * x, + ggml_tensor * attn_weights, // [src, tgt, H, B] + int64_t C, + int64_t H, + int64_t T, + int64_t B, + ggml_tensor ** tap_gated = nullptr, + ggml_tensor ** tap_attended = nullptr, + ggml_tensor ** tap_gtb = nullptr, + ggml_tensor ** tap_attnrep = nullptr, + ggml_tensor ** tap_mul = nullptr, + ggml_tensor ** tap_h = nullptr, + ggml_tensor ** tap_y2 = nullptr) { + const int64_t hidden = C * 3 / 4; + auto * h = t_linear(ctx, x, w.na_in_w.tensor, w.na_in_b.tensor); // [3*hidden, T, B] + if (tap_h != nullptr) *tap_h = h; + const size_t frame_stride = h->nb[1]; // 3*hidden floats per frame + const size_t batch_stride = h->nb[2]; + auto * s = ggml_view_3d(ctx, h, hidden, T, B, + frame_stride, batch_stride, 0); + auto * xx = ggml_view_3d(ctx, h, hidden, T, B, + frame_stride, batch_stride, hidden * sizeof(float)); + auto * y = ggml_view_3d(ctx, h, hidden, T, B, + frame_stride, batch_stride, 2 * hidden * sizeof(float)); + auto * gated = t_mul(ctx, xx, modules::TanhModule().build(ctx, value(s)).tensor); // [hidden, T, B] + if (tap_gated != nullptr) *tap_gated = gated; + + // All heads share head-0 weights, so the per-head matmul collapses to a + // single 2D contraction: out[t, c] = sum_s attn0[s, t] * gatedT[s, c], + // with gated channels already in torch (h*hd + d) order. + // out[t, c] = sum_s attn0[s, t] * gated_T[s, c]; gated channels are + // already in torch (h*hd + d) order. + auto * attn0_2d = t_cont(ctx, ggml_view_3d(ctx, attn_weights, T, T, B, + attn_weights->nb[1], attn_weights->nb[3], 0)); + if (tap_attnrep != nullptr) *tap_attnrep = attn0_2d; + ggml_tensor * gated_T; + if (B == 1) { + gated_T = t_cont(ctx, t_permute(ctx, gated, 1, 0, 2, 3)); // [s, c] + } else { + auto * g3 = t_reshape_4d(ctx, gated, hidden, T, B, 1); + gated_T = t_cont(ctx, t_permute(ctx, g3, 1, 0, 2, 3)); + } + if (tap_gtb != nullptr) *tap_gtb = gated_T; + auto * attended = t_matmul(ctx, attn0_2d, gated_T); // [t (ne0), c (ne1)] + if (tap_attended != nullptr) *tap_attended = attended; + ggml_tensor * ch2; + if (B == 1) { + ch2 = t_cont(ctx, t_permute(ctx, attended, 1, 0, 2, 3)); // [c, t] + } else { + auto * a3 = t_reshape_4d(ctx, attended, attended->ne[0], attended->ne[1], B, 1); + ch2 = t_cont(ctx, t_permute(ctx, a3, 1, 0, 2, 3)); + } + // y is a strided view; reshape would assert. Slice h directly as 2D: + // element (c, r) with r = t + b*T sits at 2*hidden + c + r*3*hidden. + auto * y2 = t_cont(ctx, ggml_view_2d(ctx, h, hidden, T * B, + h->nb[1], 2 * hidden * sizeof(float))); + if (tap_y2 != nullptr) *tap_y2 = y2; + auto * na_mul = t_mul(ctx, t_reshape_2d(ctx, ch2, hidden, T * B), y2); + if (tap_mul != nullptr) *tap_mul = na_mul; + auto * out = t_linear(ctx, na_mul, w.na_out_w.tensor, w.na_out_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// ConvolutionModule (non-causal, symmetric padding). conv_gate zeroes +// padded frames before the depthwise convolution (torch masked_fill). +ggml_tensor * conv_module( + GraphBuilder & ctx, + const ZipLayerWeights & w, + int which, + ggml_tensor * x, // [C, T, B] + ggml_tensor * conv_gate, // [1, T, 1] (0/1) + int64_t C, + int64_t T, + int64_t B) { + const auto & in_w = which == 1 ? w.cm1_in_w : w.cm2_in_w; + const auto & in_b = which == 1 ? w.cm1_in_b : w.cm2_in_b; + const auto & conv_w = which == 1 ? w.cm1_conv_w : w.cm2_conv_w; + const auto & conv_b = which == 1 ? w.cm1_conv_b : w.cm2_conv_b; + const auto & out_w = which == 1 ? w.cm1_out_w : w.cm2_out_w; + const auto & out_b = which == 1 ? w.cm1_out_b : w.cm2_out_b; + const int64_t kernel = conv_w.tensor->ne[0]; + + auto * h = t_linear(ctx, x, in_w.tensor, in_b.tensor); // [2C, T, B] + const size_t conv_frame_stride = h->nb[1]; // 2C floats per frame + const size_t conv_batch_stride = h->nb[2]; + auto * xv = ggml_view_3d(ctx, h, C, T, B, + conv_frame_stride, conv_batch_stride, 0); + auto * sv = ggml_view_3d(ctx, h, C, T, B, + conv_frame_stride, conv_batch_stride, + C * sizeof(float)); + auto * gated = t_mul(ctx, xv, modules::SigmoidModule().build(ctx, value(sv)).tensor); + gated = t_mul(ctx, gated, conv_gate); // zero padded frames + + auto input = core::wrap_tensor(gated, core::TensorShape::from_dims({B, T, C})); + input = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, input); + auto conv = modules::DepthwiseConv1dModule({C, kernel, 1, int(kernel / 2), 1, true}) + .build(ctx, input, {conv_w, conv_b}); + conv = modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, conv); + auto * with_bias = core::ensure_backend_addressable_layout(ctx, conv).tensor; + + // out_proj = SwooshR + linear + auto * activated = swoosh_r(ctx, with_bias); + auto * out = t_linear(ctx, activated, out_w.tensor, out_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// FeedforwardModule: linear -> SwooshL -> linear. +ggml_tensor * feedforward( + GraphBuilder & ctx, + const ZipLayerWeights & w, + int which, + ggml_tensor * x, + int64_t T, + int64_t B, + ggml_tensor ** inproj_out = nullptr, + ggml_tensor ** activated_out = nullptr) { + const auto & in_w = which == 1 ? w.ff1_in_w : which == 2 ? w.ff2_in_w : w.ff3_in_w; + const auto & in_b = which == 1 ? w.ff1_in_b : which == 2 ? w.ff2_in_b : w.ff3_in_b; + const auto & out_w = which == 1 ? w.ff1_out_w : which == 2 ? w.ff2_out_w : w.ff3_out_w; + const auto & out_b = which == 1 ? w.ff1_out_b : which == 2 ? w.ff2_out_b : w.ff3_out_b; + auto * h = t_linear(ctx, x, in_w.tensor, in_b.tensor); + if (inproj_out != nullptr) *inproj_out = h; + auto * activated = swoosh_l(ctx, h); + if (activated_out != nullptr) *activated_out = activated; + auto * out = t_linear(ctx, activated, out_w.tensor, out_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], T, B, 1); +} + +// Per-submodule parity taps (first text-encoder layer only). ggml tensors +// mirror torch module outputs; transpose on readback. +struct LayerTaps { + ggml_tensor * layer_in = nullptr; + ggml_tensor * attn_inproj = nullptr; // attention in_proj output + ggml_tensor * attn_qtb = nullptr; // q after permute [qh, T, H, B] + ggml_tensor * attn_ktb = nullptr; // k after permute [qh, T, H, B] + ggml_tensor * attn_scores = nullptr; // q@k before pos/bias + ggml_tensor * attn_pos = nullptr; // after pos add, before bias + ggml_tensor * attn_weights = nullptr; + ggml_tensor * ff1_inproj = nullptr; + ggml_tensor * ff1 = nullptr; + ggml_tensor * ff1_act = nullptr; + ggml_tensor * na = nullptr; + ggml_tensor * na_gated = nullptr; // after x*tanh(s) + ggml_tensor * na_gtb = nullptr; // permuted gated [T, hd, H, B] + ggml_tensor * na_attnrep = nullptr; // repeated head-0 weights + ggml_tensor * na_attended = nullptr; // after head-0 weighted matmul + ggml_tensor * na_mul = nullptr; // ch2 * y2 before out_proj + ggml_tensor * na_y2 = nullptr; // the y chunk [144, T, B] + ggml_tensor * na_h = nullptr; // in_proj output [3h, T, B] + ggml_tensor * sa1 = nullptr; + ggml_tensor * cm1_inproj = nullptr; + ggml_tensor * conv1 = nullptr; + ggml_tensor * ff2 = nullptr; + ggml_tensor * bypass_mid = nullptr; + ggml_tensor * sa2 = nullptr; + ggml_tensor * conv2 = nullptr; + ggml_tensor * ff3 = nullptr; + ggml_tensor * norm = nullptr; +}; + +// BypassModule: src_orig + (src - src_orig) * bypass_scale. +ggml_tensor * bypass(GraphBuilder & ctx, ggml_tensor * scale_t, ggml_tensor * src_orig, ggml_tensor * src) { + return modules::ScaledBypassModule().build(ctx, value(src_orig), value(src), + core::wrap_tensor(scale_t, core::TensorShape::from_dims({scale_t->ne[0]}))).tensor; +} + +// --------------------------------------------------------------------------- +// encoder layer / stack / zipformer + +ggml_tensor * encoder_layer( + GraphBuilder & ctx, + Consts & consts, + const ZipLayerWeights & w, + ggml_tensor * src, // [C, T, B] + ggml_tensor * attn_bias, // [T, 1, 1, 1] + ggml_tensor * conv_gate, // [1, T, 1] + ggml_tensor * time_row, // [C, 1, 1] or null + const ZipVoiceConfig & config, + int64_t kernel, + int64_t T, + int64_t B, + bool fm, + LayerTaps * taps = nullptr) { + const int64_t C = src->ne[0]; + const int64_t H = fm ? config.fm_num_heads : config.text_num_heads; + + auto * src_orig = src; + if (taps != nullptr) taps->layer_in = src; + ggml_tensor * attn_qtb = nullptr; + ggml_tensor * attn_ktb = nullptr; + ggml_tensor * attn_inproj = nullptr; + ggml_tensor * attn_scores = nullptr; + ggml_tensor * attn_pos = nullptr; + auto * attn_weights = attention_weights( + ctx, consts, w, src, attn_bias, H, config.query_head_dim, + config.pos_head_dim, config.pos_dim, T, B, + taps != nullptr ? &attn_qtb : nullptr, + taps != nullptr ? &attn_ktb : nullptr, + taps != nullptr ? &attn_inproj : nullptr, + taps != nullptr ? &attn_scores : nullptr, + taps != nullptr ? &attn_pos : nullptr); + if (taps != nullptr) { + taps->attn_qtb = attn_qtb; + taps->attn_ktb = attn_ktb; + taps->attn_inproj = attn_inproj; + taps->attn_scores = attn_scores; + taps->attn_pos = attn_pos; + taps->attn_weights = attn_weights; + } + + auto * x = src; + if (time_row != nullptr) { + x = t_add(ctx, x, time_row); + } + ggml_tensor * ff1_inproj = nullptr; + ggml_tensor * ff1_act = nullptr; + auto * ff1_out = feedforward(ctx, w, 1, x, T, B, &ff1_inproj, &ff1_act); + if (taps != nullptr) { taps->ff1_inproj = ff1_inproj; taps->ff1_act = ff1_act; taps->ff1 = ff1_out; } + x = t_add(ctx, x, ff1_out); + + // nonlin attention uses only head 0 + ggml_tensor * na_gated = nullptr; + ggml_tensor * na_attended = nullptr; + ggml_tensor * na_gtb = nullptr; + ggml_tensor * na_attnrep = nullptr; + ggml_tensor * na_mul = nullptr; + ggml_tensor * na_h = nullptr; + ggml_tensor * na_y2 = nullptr; + auto * na_out = nonlin_attention(ctx, w, x, attn_weights, C, H, T, B, + &na_gated, &na_attended, &na_gtb, &na_attnrep, + &na_mul, &na_h, &na_y2); + if (taps != nullptr) { + taps->na = na_out; + taps->na_gated = na_gated; + taps->na_gtb = na_gtb; + taps->na_attnrep = na_attnrep; + taps->na_attended = na_attended; + taps->na_mul = na_mul; + taps->na_y2 = na_y2; + taps->na_h = na_h; + } + x = t_add(ctx, x, na_out); + + auto * sa1_out = self_attention(ctx, w, 1, x, attn_weights, H, config.value_head_dim, T, B); + if (taps != nullptr) taps->sa1 = sa1_out; + x = t_add(ctx, x, sa1_out); + + if (time_row != nullptr) { + x = t_add(ctx, x, time_row); + } + // conv in_proj runs on the pre-conv activation (torch cm1_inproj input) + auto * cm1_out = conv_module(ctx, w, 1, x, conv_gate, C, T, B); + if (taps != nullptr) taps->conv1 = cm1_out; + x = t_add(ctx, x, cm1_out); + auto * ff2_out = feedforward(ctx, w, 2, x, T, B); + if (taps != nullptr) taps->ff2 = ff2_out; + x = t_add(ctx, x, ff2_out); + + x = bypass(ctx, w.bypass_mid_scale.tensor, src_orig, x); + if (taps != nullptr) taps->bypass_mid = x; + + auto * sa2_out = self_attention(ctx, w, 2, x, attn_weights, H, config.value_head_dim, T, B); + if (taps != nullptr) taps->sa2 = sa2_out; + x = t_add(ctx, x, sa2_out); + + if (time_row != nullptr) { + x = t_add(ctx, x, time_row); + } + auto * conv2_out = conv_module(ctx, w, 2, x, conv_gate, C, T, B); + if (taps != nullptr) taps->conv2 = conv2_out; + x = t_add(ctx, x, conv2_out); + auto * ff3_out = feedforward(ctx, w, 3, x, T, B); + if (taps != nullptr) taps->ff3 = ff3_out; + x = t_add(ctx, x, ff3_out); + + x = bias_norm(ctx, x, w.norm_bias.tensor, w.norm_log_scale); + if (taps != nullptr) taps->norm = x; + x = bypass(ctx, w.bypass_scale.tensor, src_orig, x); + return x; +} + +// SimpleDownsample: softmax-weighted sum over ds frames; pads by repeating +// the last frame. Input [C, T, B] -> [C, ceil(T/ds), B]. +ggml_tensor * downsample( + GraphBuilder & ctx, + Consts & consts, + const std::vector & softmax_bias, + ggml_tensor * x, + int64_t T, + int64_t B) { + const int64_t ds = static_cast(softmax_bias.size()); + const int64_t Td = (T + ds - 1) / ds; + const int64_t Tp = Td * ds; + const int64_t C = x->ne[0]; + + auto * x2d = t_reshape_2d(ctx, x, C, T * B); // rows (t, b) t-fastest + std::vector idx(static_cast(Tp * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t j = 0; j < Tp; ++j) { + idx[static_cast(j + b * Tp)] = + static_cast(std::min(j, T - 1) + b * T); + } + } + auto * gather = consts.i32(idx, {Tp * B}); + auto * padded = t_rows(ctx, x2d, gather); // [C, Tp*B] + + // weighted sum over the ds sub-frame axis + auto * windowed = t_reshape_4d(ctx, padded, C, ds, Td, B); + auto * w_frame = t_permute(ctx, windowed, 3, 0, 1, 2); // [ds, Td, B, C] + auto * w_frame_c = t_cont(ctx, w_frame); + auto * w2 = t_reshape_3d(ctx, w_frame_c, ds, Td * B, C); + auto * weights2 = consts.f32(softmax_bias, {ds, 1}); + auto * summed = t_matmul(ctx, weights2, w2); // [1, Td*B, C] + auto * as_c = t_permute(ctx, summed, 1, 2, 0, 3); // [C, 1, Td*B, 1] + auto * as_c2 = t_cont(ctx, as_c); + auto * out = t_reshape_3d(ctx, as_c2, C, Td, B); + return out; +} + +// SimpleUpsample: repeat each frame ds times, then truncate to T. +ggml_tensor * upsample( + GraphBuilder & ctx, + Consts & consts, + int64_t ds, + int64_t T, + ggml_tensor * x, // [C, Td, B] + int64_t B) { + const int64_t Td = x->ne[1]; + const int64_t C = x->ne[0]; + auto * x2d = t_reshape_2d(ctx, x, C, Td * B); + const int64_t Tp = Td * ds; + std::vector idx(static_cast(Tp * B)); + for (int64_t b = 0; b < B; ++b) { + for (int64_t i = 0; i < Tp; ++i) { + idx[static_cast(i + b * Tp)] = + static_cast(i / ds + b * Td); + } + } + auto * gather = consts.i32(idx, {Tp * B}); + auto * out = t_rows(ctx, x2d, gather); // [C, Tp*B] contiguous + if (Tp == T) { + return t_reshape_3d(ctx, out, C, T, B); + } + // truncate to T frames: rows are (frame, batch) frame-fastest + return ggml_view_3d(ctx, out, C, T, B, + C * sizeof(float), static_cast(Tp) * C * sizeof(float), 0); +} + +ggml_tensor * zipformer_forward( + GraphBuilder & ctx, + Consts & consts, + const TTSZipformerWeights & w, + const ZipVoiceConfig & config, + ggml_tensor * input, // [in_dim, T, B] + ggml_tensor * time_emb, // [time_embed_dim] or null + ggml_tensor * const * pad_bias, + ggml_tensor * const * conv_gate, + bool fm, + int64_t B, + std::vector * layer_taps = nullptr, + LayerTaps * first_layer_taps = nullptr) { + const bool with_time = fm; + auto * x = t_linear(ctx, input, w.in_proj_w.tensor, w.in_proj_b.tensor); // [C, T, B] + + ggml_tensor * outer_time = nullptr; + if (with_time && time_emb != nullptr) { + auto * h = t_linear(ctx, time_emb, w.time_mlp0_w.tensor, w.time_mlp0_b.tensor); + outer_time = t_linear(ctx, swoosh_r(ctx, h), w.time_mlp2_w.tensor, w.time_mlp2_b.tensor); + } + + const int64_t T = x->ne[1]; + for (size_t s = 0; s < w.stacks.size(); ++s) { + auto & stack = w.stacks[s]; + const int64_t ds = fm ? config.fm_downsampling_factor[s] : 1; + const int64_t kernel = fm ? config.fm_cnn_kernel[s] : config.text_cnn_kernel; + + ggml_tensor * time_row = nullptr; + if (with_time && stack.time_proj_w.tensor != nullptr) { + // torch: encoders[i].time_emb = Sequential(SwooshR, Linear) + auto * activated = swoosh_r(ctx, outer_time); + auto * projected = t_linear(ctx, activated, stack.time_proj_w.tensor, stack.time_proj_b.tensor); + time_row = t_reshape_3d(ctx, projected, projected->ne[0], 1, 1); + } + + ggml_tensor * stack_in = x; + if (ds > 1) { + stack_in = downsample(ctx, consts, stack.downsample_bias, x, T, B); + } + const int64_t Ts = stack_in->ne[1]; + + auto * h = stack_in; + for (size_t li = 0; li < stack.layers.size(); ++li) { + h = encoder_layer( + ctx, consts, stack.layers[li], h, pad_bias[s], conv_gate[s], time_row, + config, kernel, Ts, B, fm, + first_layer_taps != nullptr && s == 0 && li == 0 ? first_layer_taps : nullptr); + if (layer_taps != nullptr) { + layer_taps->push_back(h); + } + } + + if (ds > 1) { + auto * up = upsample(ctx, consts, ds, T, h, B); + x = bypass(ctx, stack.out_combiner_scale.tensor, x, up); + } else { + x = h; + } + } + auto * out = t_linear(ctx, x, w.out_proj_w.tensor, w.out_proj_b.tensor); + return t_reshape_4d(ctx, out, out->ne[0], out->ne[1], B, 1); +} + +struct GraphBuildResult { + ggml_context * ctx = nullptr; + ggml_cgraph * graph = nullptr; +}; + +GraphBuildResult finish_graph( + ggml_context * ctx, + ggml_context * tensor_ctx, + ggml_tensor * output, + const std::vector & extra_roots, + std::vector & staged, + size_t node_budget, + ggml_backend_t backend, + void ** gallocr_out, + ggml_backend_buffer_t * buffer_out) { + GraphBuildResult result; + result.ctx = ctx; + result.graph = ggml_new_graph_custom(ctx, node_budget, false); + ggml_build_forward_expand(result.graph, output); + // ALL extra outputs must be expanded BEFORE the arena reservation: + // nodes appended after ggml_gallocr_alloc_graph never get memory. + for (ggml_tensor * tap : extra_roots) { + if (tap != nullptr) { + ggml_build_forward_expand(result.graph, tap); + } + } + if (backend == nullptr) { + throw std::runtime_error("zipvoice: graph backend is required"); + } + // Half-staged matrix products accumulate audible error across the flow + // solver steps. Keep the CPU oracle unchanged and request F32 on GPUs. + if (core::backend_type(backend) != core::BackendType::Cpu) { + for (int i = 0; i < ggml_graph_n_nodes(result.graph); ++i) { + auto * node = ggml_graph_node(result.graph, i); + if (node->op == GGML_OP_MUL_MAT) ggml_mul_mat_set_prec(node, GGML_PREC_F32); + } + } + core::validate_backend_graph_supported(backend, result.graph, "zipvoice"); + // private storage for leaves + constants FIRST: with a buffer already + // assigned, gallocr skips these tensors instead of arena-aliasing them + ggml_backend_buffer_t buffer = + ggml_backend_alloc_ctx_tensors(tensor_ctx, backend); + if (buffer == nullptr) { + throw std::runtime_error("zipvoice: tensor buffer allocation failed"); + } + *buffer_out = buffer; + auto * gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + *gallocr_out = gallocr; + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, result.graph) || + !ggml_gallocr_alloc_graph(gallocr, result.graph)) { + throw std::runtime_error("zipvoice: graph allocation failed"); + } + for (auto & item : staged) { + ggml_backend_tensor_set(item.tensor, item.bytes.data(), 0, item.bytes.size()); + } + staged.clear(); + return result; +} + +} // namespace + +FmDecoderGraph build_fm_decoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t T, + int64_t B, + bool with_guidance, + bool cuda_backend, + ggml_backend_t backend) { + (void)cuda_backend; + const size_t ctx_bytes = 512ULL * 1024ULL * 1024ULL; + ggml_context * ctx = ggml_init({ctx_bytes, nullptr, true}); + // leaves + constants get their own context and backend buffer so the + // graph arena (gallocr) never aliases them (see skill: gallocr aliasing) + ggml_context * tensor_ctx = ggml_init({64ULL * 1024ULL * 1024ULL, nullptr, true}); + FmDecoderGraph g; + g.ctx = ctx; + g.tensor_ctx = tensor_ctx; + g.T = T; + g.B = B; + g.cuda = false; + g.backend = backend; + + std::vector staged; + Consts consts{tensor_ctx, &staged}; + + auto * x_cat = ggml_new_tensor_3d(tensor_ctx, GGML_TYPE_F32, config.feat_dim * 3, T, B); + ggml_set_input(x_cat); + g.x_cat = x_cat; + + auto * time_emb = ggml_new_tensor_1d(tensor_ctx, GGML_TYPE_F32, config.time_embed_dim); + ggml_set_input(time_emb); + g.time_emb = time_emb; + + if (with_guidance) { + auto * guidance = ggml_new_tensor_1d(tensor_ctx, GGML_TYPE_F32, config.time_embed_dim); + ggml_set_input(guidance); + g.guidance_emb = guidance; + } + + // per-stack masks. Downsampling factors are relative to the full frame + // rate (U-Net style), so stack s runs at ceil(T / ds[s]) frames. + for (size_t s = 0; s < weights.fm_decoder.stacks.size(); ++s) { + const int64_t ds = config.fm_downsampling_factor[s]; + const int64_t T_s = (T + ds - 1) / ds; + auto * bias = ggml_new_tensor_4d(tensor_ctx, GGML_TYPE_F32, T_s, 1, 1, 1); + ggml_set_input(bias); + g.pad_bias[s] = bias; + auto * gate = ggml_new_tensor_3d(tensor_ctx, GGML_TYPE_F32, 1, T_s, 1); + ggml_set_input(gate); + g.conv_gate[s] = gate; + } + + GraphBuilder builder(ctx, core::backend_type(backend)); + ggml_tensor * time_input = g.time_emb; + if (with_guidance && weights.fm_decoder.guidance_embed_w.tensor != nullptr) { + auto * guidance_proj = t_linear( + builder, g.guidance_emb, weights.fm_decoder.guidance_embed_w.tensor, nullptr); + time_input = t_add(builder, g.time_emb, guidance_proj); + } + + auto * out = zipformer_forward( + builder, consts, weights.fm_decoder, config, x_cat, time_input, + g.pad_bias, g.conv_gate, true, B); + ggml_set_output(out); + g.output = out; + + auto built = finish_graph(ctx, tensor_ctx, out, {}, staged, 262144, backend, &g.gallocr, &g.buffer); + g.graph = built.graph; + return g; +} + +TextEncoderGraph build_text_encoder_graph( + const ZipVoiceWeights & weights, + const ZipVoiceConfig & config, + int64_t S, + bool cuda_backend, + ggml_backend_t backend) { + (void)cuda_backend; + const size_t ctx_bytes = 256ULL * 1024ULL * 1024ULL; + ggml_context * ctx = ggml_init({ctx_bytes, nullptr, true}); + ggml_context * tensor_ctx = ggml_init({16ULL * 1024ULL * 1024ULL, nullptr, true}); + TextEncoderGraph g; + g.ctx = ctx; + g.tensor_ctx = tensor_ctx; + g.S = S; + g.cuda = false; + g.backend = backend; + + std::vector staged; + Consts consts{tensor_ctx, &staged}; + + auto * ids = ggml_new_tensor_1d(tensor_ctx, GGML_TYPE_I32, S); + ggml_set_input(ids); + g.token_ids = ids; + + auto * bias = ggml_new_tensor_4d(tensor_ctx, GGML_TYPE_F32, S, 1, 1, 1); + ggml_set_input(bias); + g.pad_bias[0] = bias; + auto * gate = ggml_new_tensor_3d(tensor_ctx, GGML_TYPE_F32, 1, S, 1); + ggml_set_input(gate); + g.conv_gate[0] = gate; + + GraphBuilder builder(ctx, core::backend_type(backend)); + auto * embedded = t_rows(builder, weights.text_encoder.embed_w.tensor, ids); + std::vector layer_taps; + LayerTaps first_layer; + auto * out = zipformer_forward( + builder, consts, weights.text_encoder, config, embedded, nullptr, + g.pad_bias, g.conv_gate, false, 1, &layer_taps, &first_layer); + ggml_set_output(out); + g.output = out; + std::vector extra_roots; + const auto mark_output = [](ggml_tensor * tap) { + ggml_set_output(tap); + // reshape results are VIEWS: the gallocr frees/reuses the underlying + // node unless it is flagged too (graph outputs are never freed). + if (tap->view_src != nullptr) { + ggml_set_output(tap->view_src); + } + }; + for (auto * tap : layer_taps) { + mark_output(tap); + g.layer_taps.push_back(tap); + extra_roots.push_back(tap); + } + for (ggml_tensor * tap : { + first_layer.layer_in, first_layer.attn_qtb, first_layer.attn_ktb, + first_layer.attn_inproj, first_layer.attn_scores, + first_layer.attn_pos, first_layer.attn_weights, first_layer.ff1_inproj, + first_layer.ff1_act, first_layer.ff1, first_layer.na, first_layer.na_gated, + first_layer.na_gtb, first_layer.na_attnrep, first_layer.na_attended, first_layer.na_mul, first_layer.na_y2, first_layer.na_h, first_layer.sa1, first_layer.cm1_inproj, + first_layer.conv1, first_layer.ff2, first_layer.bypass_mid, + first_layer.sa2, first_layer.conv2, first_layer.ff3, first_layer.norm}) { + if (tap != nullptr) { + mark_output(tap); + extra_roots.push_back(tap); + } + } + + auto built = finish_graph(ctx, tensor_ctx, out, extra_roots, staged, 262144, backend, &g.gallocr, &g.buffer); + g.stage_taps.clear(); + for (ggml_tensor * tap : { + first_layer.layer_in, first_layer.attn_qtb, first_layer.attn_ktb, + first_layer.attn_inproj, first_layer.attn_scores, + first_layer.attn_pos, first_layer.attn_weights, first_layer.ff1_inproj, + first_layer.ff1_act, first_layer.ff1, first_layer.na, first_layer.na_gated, + first_layer.na_gtb, first_layer.na_attnrep, first_layer.na_attended, first_layer.na_mul, first_layer.na_y2, first_layer.na_h, first_layer.sa1, first_layer.cm1_inproj, + first_layer.conv1, first_layer.ff2, first_layer.bypass_mid, + first_layer.sa2, first_layer.conv2, first_layer.ff3, first_layer.norm}) { + if (tap != nullptr) g.stage_taps.push_back(tap); + } + g.graph = built.graph; + return g; +} + +} // namespace engine::models::zipvoice diff --git a/src/framework/audio/espeak_phonemizer.cpp b/src/framework/audio/espeak_phonemizer.cpp index 58c2b995b..e8f895144 100644 --- a/src/framework/audio/espeak_phonemizer.cpp +++ b/src/framework/audio/espeak_phonemizer.cpp @@ -79,8 +79,13 @@ struct Runtime { #ifdef _WIN32 "espeak-ng.dll", "libespeak-ng.dll", #elif defined(__APPLE__) + "/opt/homebrew/lib/libespeak-ng.dylib", + "/usr/local/lib/libespeak-ng.dylib", "libespeak-ng.dylib", "libespeak-ng.1.dylib", #else + "/usr/lib/x86_64-linux-gnu/libespeak-ng.so.1", + "/usr/lib/aarch64-linux-gnu/libespeak-ng.so.1", + "/usr/lib/libespeak-ng.so.1", "libespeak-ng.so.1", "libespeak-ng.so", #endif }); @@ -130,6 +135,27 @@ EspeakPhonemizer::EspeakPhonemizer(std::filesystem::path library, #endif if (data_.extension() == ".bin" || data_.extension() == ".gguf") data_ = materialize_espeak_data(data_); +#if !defined(AUDIOCPP_STATIC_ESPEAK) && !defined(_WIN32) + // Dynamic eSpeak-ng with no explicit data path: probe common install + // locations (library-relative first, then system share directories) so a + // stock install works without session options. + if (data_.empty()) { + std::vector data_candidates; + if (!library_.empty()) { + const auto lib_dir = library_.parent_path(); + data_candidates.push_back(lib_dir / ".." / "share" / "espeak-ng-data"); + data_candidates.push_back(lib_dir / "espeak-ng-data"); + } + data_candidates.push_back("/opt/homebrew/share/espeak-ng-data"); + data_candidates.push_back("/usr/local/share/espeak-ng-data"); + data_candidates.push_back("/usr/share/espeak-ng-data"); + for (const auto & candidate : data_candidates) + if (std::filesystem::is_regular_file(candidate / "phontab")) { + data_ = candidate.lexically_normal(); + break; + } + } +#endif if (!data_.empty() && data_.filename().empty()) data_ = data_.parent_path(); if (!library_.empty() && !std::filesystem::is_regular_file(library_)) throw std::runtime_error("eSpeak-ng library does not exist: " + library_.string()); diff --git a/src/framework/modules/convnext_modules.cpp b/src/framework/modules/convnext_modules.cpp new file mode 100644 index 000000000..b4d29d607 --- /dev/null +++ b/src/framework/modules/convnext_modules.cpp @@ -0,0 +1,23 @@ +#include "engine/framework/modules/convnext_modules.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" + +namespace engine::modules { +core::TensorValue build_convnext1d(core::ModuleBuildContext & ctx, + const core::TensorValue & input, const ConvNeXt1dWeights & weights) { + core::validate_rank_between(input, 3, 3, "ConvNeXt1d input"); + const int64_t channels = input.shape.last_dim(); + const int64_t kernel = weights.depthwise.weight.shape.last_dim(); + const int64_t hidden = weights.expansion.weight.shape.at(0); + auto x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, input); + x = DepthwiseConv1dModule({channels, kernel, 1, int(kernel / 2), 1, true}).build(ctx, x, weights.depthwise); + x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, x); + x = LayerNormModule({channels, 1e-6f, true, true}).build(ctx, x, weights.norm); + x = LinearModule({channels, hidden, true}).build(ctx, x, weights.expansion); + x = GeluModule({GeluApproximation::ExactErf}).build(ctx, x); + x = LinearModule({hidden, channels, true}).build(ctx, x, weights.projection); + x = MulModule().build(ctx, x, RepeatModule({x.shape}).build(ctx, core::reshape_tensor(ctx, weights.gamma, core::TensorShape::from_dims({1, 1, channels})))); + return AddModule().build(ctx, input, x); +} +} diff --git a/src/framework/modules/vocoders/vocos_vocoder.cpp b/src/framework/modules/vocoders/vocos_vocoder.cpp new file mode 100644 index 000000000..819120574 --- /dev/null +++ b/src/framework/modules/vocoders/vocos_vocoder.cpp @@ -0,0 +1,143 @@ +#include "engine/framework/modules/vocoders/vocos_vocoder.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/audio/istft_graph.h" +#include "engine/framework/modules/convnext_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/runtime/cache_slots.h" +#include "ggml-alloc.h" +#include +#include +#include + +namespace engine::modules { +namespace { +constexpr int kMel = 100, kFFT = 1024, kHop = 256; +struct Graph { + ggml_backend_t backend = nullptr; + ggml_context * ctx = nullptr; + ggml_context * inputs = nullptr; + ggml_backend_buffer_t buffer = nullptr; + ggml_gallocr_t arena = nullptr; + ggml_cgraph * graph = nullptr; + ggml_tensor * mel = nullptr; + ggml_tensor * spec = nullptr; + std::unique_ptr istft; + ~Graph() { + if (backend && graph) core::release_backend_graph_resources(backend, graph, true); + if (arena) ggml_gallocr_free(arena); + if (buffer) ggml_backend_buffer_free(buffer); + if (ctx) ggml_free(ctx); + if (inputs) ggml_free(inputs); + } +}; +} +core::TensorValue build_vocos_backbone(core::ModuleBuildContext & ctx, + const core::TensorValue & mel, const VocosBackboneWeights & weights) { + const auto channels = weights.embed.weight.shape.at(0); + const auto kernel = weights.embed.weight.shape.last_dim(); + auto x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, mel); + x = Conv1dModule({mel.shape.last_dim(), channels, kernel, 1, int(kernel / 2), 1, true}).build(ctx, x, weights.embed); + x = TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, x); + x = LayerNormModule({channels, 1e-6f, true, true}).build(ctx, x, weights.input_norm); + for (const auto & block : weights.blocks) x = build_convnext1d(ctx, x, block); + x = LayerNormModule({channels, 1e-6f, true, true}).build(ctx, x, weights.final_norm); + return LinearModule({channels, weights.head.weight.shape.at(0), true}).build(ctx, x, weights.head); +} +class VocosVocoder::Impl { +public: + ggml_backend_t backend; + int threads; + core::BackendType type; + core::BackendWeightStore store; + VocosBackboneWeights weights; + std::vector window; + runtime::CacheSlots> graphs{1}; + + Impl(const std::string & checkpoint, ggml_backend_t b, int n) + : backend(b), threads(n > 0 ? n : 1), type(core::backend_type(b)), + store(b, type, "vocos", 8ULL << 20) { + auto source = assets::open_tensor_source(checkpoint); + if (std::filesystem::path(checkpoint).extension() == ".gguf") { + source = assets::make_prefixed_tensor_source(source, "vocos"); + } + const auto f32 = [&](const std::string & name) { + return store.load_f32_tensor(*source, name, source->require_metadata(name).shape); + }; + const auto norm = [&](const std::string & name) -> NormWeights { + return {f32(name + ".weight"), f32(name + ".bias")}; + }; + const auto linear = [&](const std::string & name) -> LinearWeights { + return {f32(name + ".weight"), f32(name + ".bias")}; + }; + weights.embed = {f32("backbone.embed.weight"), f32("backbone.embed.bias")}; + weights.input_norm = norm("backbone.norm"); + weights.final_norm = norm("backbone.final_layer_norm"); + weights.head = linear("head.out"); + for (int i = 0; source->has_tensor("backbone.convnext." + std::to_string(i) + ".dwconv.weight"); ++i) { + const auto p = "backbone.convnext." + std::to_string(i); + weights.blocks.push_back({{f32(p + ".dwconv.weight"), f32(p + ".dwconv.bias")}, + norm(p + ".norm"), linear(p + ".pwconv1"), linear(p + ".pwconv2"), f32(p + ".gamma")}); + } + if (weights.blocks.empty()) throw std::runtime_error("Vocos has no ConvNeXt blocks"); + store.upload(); + source->release_storage(); + window.resize(kFFT); + for (int i = 0; i < kFFT; ++i) window[i] = 0.5f * (1 - std::cos(2.0f * 3.14159265358979323846f * i / kFFT)); + } + + std::unique_ptr build(int64_t frames) { + auto g = std::make_unique(); + g->backend = backend; + g->ctx = ggml_init({16ULL << 20, nullptr, true}); + g->inputs = ggml_init({1ULL << 20, nullptr, true}); + if (!g->ctx || !g->inputs) throw std::runtime_error("Vocos context allocation failed"); + core::ModuleBuildContext ctx{g->ctx, "vocos", type}, io{g->inputs, "vocos.input", type}; + auto mel = core::make_tensor(io, GGML_TYPE_F32, core::TensorShape::from_dims({1, frames, kMel})); + g->mel = mel.tensor; + ggml_set_input(g->mel); + auto x = build_vocos_backbone(ctx, mel, weights); + g->spec = x.tensor; + ggml_set_output(g->spec); + g->graph = ggml_new_graph_custom(g->ctx, 8192, false); + ggml_build_forward_expand(g->graph, g->spec); + if (type != core::BackendType::Cpu) { + for (int i = 0; i < ggml_graph_n_nodes(g->graph); ++i) { + auto * node = ggml_graph_node(g->graph, i); + if (node->op == GGML_OP_MUL_MAT) ggml_mul_mat_set_prec(node, GGML_PREC_F32); + } + } + g->buffer = ggml_backend_alloc_ctx_tensors(g->inputs, backend); + g->arena = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (!g->buffer || !g->arena || !ggml_gallocr_alloc_graph(g->arena, g->graph)) + throw std::runtime_error("Vocos graph allocation failed"); + g->istft = std::make_unique( + audio::HostLogMagnitudePhaseISTFTConfig{frames, kFFT, kHop, kFFT + 2, size_t(threads)}); + return g; + } + std::vector decode(const std::vector & mel) { + if (mel.size() < 2 * kMel || mel.size() % kMel) throw std::invalid_argument("Vocos needs complete 100-bin mel frames"); + const int64_t frames = mel.size() / kMel; + if (!graphs.find(frames)) { + // Release the previous arena before allocating the next shape. + graphs.clear(); + graphs.put(frames, build(frames)); + } + auto & g = **graphs.find(frames); + ggml_backend_tensor_set(g.mel, mel.data(), 0, mel.size() * sizeof(float)); + core::set_backend_threads(backend, threads); + if (core::compute_backend_graph(backend, g.graph) != GGML_STATUS_SUCCESS) throw std::runtime_error("Vocos compute failed"); + ggml_backend_synchronize(backend); + auto spec = core::read_tensor_f32(g.spec); + auto audio = g.istft->compute(spec, window).audio; + // Framework ISTFT uses same padding (T*hop). Vocos uses center=True: + // trim a further half hop at each end to obtain (T-1)*hop samples. + return {audio.begin() + kHop / 2, audio.end() - kHop / 2}; + } +}; +VocosVocoder::VocosVocoder(const std::string & path, ggml_backend_t backend, int threads) + : impl_(std::make_unique(path, backend, threads)) {} +VocosVocoder::~VocosVocoder() = default; +std::vector VocosVocoder::decode(const std::vector & mel) { return impl_->decode(mel); } +size_t VocosVocoder::cached_graph_count() const { return impl_->graphs.size(); } +} diff --git a/tests/zipvoice/build_reference.py b/tests/zipvoice/build_reference.py new file mode 100644 index 000000000..9eba604aa --- /dev/null +++ b/tests/zipvoice/build_reference.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Generate parity references from an upstream ZipVoice checkout and fixed inputs. + +No downloads or text frontend are needed. Run with the upstream Python environment: + python build_reference.py --repo /path/to/ZipVoice --model-dir /path/to/zipvoice_distill \ + --fixture moonlight_distill_case.npz --output reference.npz [--vocos model.safetensors] + +Hooks capture actual masked module outputs, never a hand-recomputed approximation. +The supplied fixture's text_condition/x1 are retained as independent golden outputs. +""" +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +import torch + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--fixture", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--vocos", type=Path) + args = parser.parse_args() + sys.path.insert(0, str(args.repo.resolve())) + from zipvoice.models.zipvoice_distill import ZipVoiceDistill + from zipvoice.utils.feature import VocosFbank + + torch.set_num_threads(4) + config = json.loads((args.model_dir / "model.json").read_text()) + vocab = dict(line.rstrip("\n").split("\t") for line in + (args.model_dir / "tokens.txt").read_text().splitlines()) + model = ZipVoiceDistill(**config["model"], vocab_size=len(vocab), pad_id=int(vocab["_"])) + state = torch.load(args.model_dir / "model.pt", map_location="cpu", weights_only=False)["model"] + model.load_state_dict(state, strict=True) + model.eval() + with np.load(args.fixture) as fixture: + outputs = {key: fixture[key] for key in fixture.files} + + def capture(name): + def hook(module, inputs, output): + outputs[name] = output.detach().clone().cpu().numpy().astype(np.float32) + return hook + + hooks = [] + for i, layer in enumerate(model.text_encoder.encoders[0].layers): + hooks.append(layer.register_forward_hook(capture(f"layer{i}"))) + first = model.text_encoder.encoders[0].layers[0] + for name in ("self_attn_weights", "feed_forward1", "nonlin_attention", "self_attn1", + "conv_module1", "feed_forward2", "bypass_mid", "self_attn2", + "conv_module2", "feed_forward3", "norm"): + hooks.append(dict(first.named_modules())[name].register_forward_hook(capture(name))) + for name, key in (("nonlin_attention.in_proj", "na_h_ref"), + ("nonlin_attention.identity1", "na_gated_ref"), + ("nonlin_attention.identity2", "na_y2_ref"), + ("nonlin_attention.identity3", "na_mul_ref")): + hooks.append(dict(first.named_modules())[name].register_forward_hook(capture(key))) + with torch.inference_mode(): + tokens = outputs["tokens"].tolist() + prompt_tokens = outputs["prompt_tokens"].tolist() + embed, _ = model.forward_text_embed([prompt_tokens[0] + tokens[0]]) + outputs["embed"] = embed.numpy() + for hook in hooks: + hook.remove() + tc, mask = model.forward_text_inference_ratio_duration( + tokens=tokens, prompt_tokens=prompt_tokens, + prompt_features_lens=torch.from_numpy(outputs["prompt_features_lens"]), speed=1.0) + np.testing.assert_allclose(tc.numpy(), outputs["text_condition"], atol=2e-6, rtol=2e-5) + # Odd and even lengths exercise repeat-last padding and downsampling. + for length in (17, 32, outputs["x0"].shape[1]): + for step, time in enumerate((0.0, 0.6)): + key = f"velocity_{length}_{step}" + result = model.forward_fm_decoder( + t=torch.tensor(time), xt=torch.from_numpy(outputs["x0"][:, :length]), + text_condition=tc[:, :length], + speech_condition=torch.from_numpy(outputs["speech_condition"][:, :length]), + padding_mask=mask[:, :length], guidance_scale=torch.tensor(3.0)) + outputs[key] = result.numpy() + # Different conditions in each batch catch accidental head-0/batch-0 reuse. + length = 17 + batch_x = torch.from_numpy(outputs["x0"][:, :length]).repeat(2, 1, 1) + batch_text = torch.cat([torch.zeros_like(tc[:, :length]), tc[:, :length]]) + batch_speech = torch.from_numpy(outputs["speech_condition"][:, :length]).repeat(2, 1, 1) + outputs["batch_x"] = batch_x.numpy() + outputs["batch_text"] = batch_text.numpy() + outputs["batch_speech"] = batch_speech.numpy() + outputs["batch_velocity"] = model.forward_fm_decoder( + t=torch.tensor(0.6), xt=batch_x, text_condition=batch_text, + speech_condition=batch_speech, padding_mask=mask[:, :length].repeat(2, 1), + guidance_scale=torch.tensor(3.0)).numpy() + # Broadband input keeps quiet high-frequency bins numerically well conditioned. + wav = torch.from_numpy(np.random.default_rng(123).normal(0, 0.1, 4096).astype(np.float32)) + outputs["fbank_wav"] = wav.numpy() + outputs["fbank_mel"] = VocosFbank().extract(wav, sampling_rate=24000).numpy() + if args.vocos: + from safetensors.torch import load_file + from vocos.models import VocosBackbone + from vocos.heads import ISTFTHead + backbone = VocosBackbone(input_channels=100, dim=512, intermediate_dim=1536, num_layers=8).eval() + head = ISTFTHead(dim=512, n_fft=1024, hop_length=256, padding="center").eval() + weights = (load_file(str(args.vocos)) if args.vocos.suffix == ".safetensors" + else torch.load(args.vocos, map_location="cpu", weights_only=True)) + backbone.load_state_dict({k.removeprefix("backbone."): v for k, v in weights.items() if k.startswith("backbone.")}) + head.load_state_dict({k.removeprefix("head."): v for k, v in weights.items() if k.startswith("head.")}) + # Skip the leading silence in the reference recording. + mel = torch.from_numpy(outputs["prompt_features"][:, 32:64]) / 0.1 + outputs["vocos_mel"] = mel.numpy() + outputs["vocos_audio"] = head(backbone(mel.transpose(1, 2))).numpy() + assert np.sqrt(np.mean(outputs["vocos_audio"] ** 2)) > 1e-3, "Vocos fixture must contain speech" + args.output.parent.mkdir(parents=True, exist_ok=True) + np.savez(args.output, **outputs) + print(f"Wrote {args.output}: {len(outputs)} arrays") + + +if __name__ == "__main__": + main() diff --git a/tests/zipvoice/zh_reference_ids.py b/tests/zipvoice/zh_reference_ids.py new file mode 100644 index 000000000..3824eecde --- /dev/null +++ b/tests/zipvoice/zh_reference_ids.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Dump reference EmiliaTokenizer token ids for the zh-token parity test. + +Run with the upstream ZipVoice Python environment: + /path/to/ZipVoice/.venv/bin/python tests/zipvoice/zh_reference_ids.py \ + --tokenizer-repo /path/to/ZipVoice \ + --model-dir /models/ZipVoice/zipvoice_distill \ + --output /tmp/zipvoice/zh_reference_ids.json +""" +import argparse +import json +import sys +from pathlib import Path + + +CORPUS = [ + "今夜的月光如此清亮，不做些什么真是浪费。随我一同去月下漫步吧，不许拒绝。", + "你好，世界。", + "老虎养殖场里养着几只老虎。", + "一点诚意都没有，不见不散。", + "一直走，不要停，第一个路口右转。", + "我在银行门口的长凳上休息了很久。", + "重庆的重量单位和别处不一样吗？", + "2024年3月14日，第2名，50元。", + "我用 audio.cpp 合成语音，效果不错！", + "This is English text inside.", + " 人最棒。", + "他拿着画，走向画室。", + "买东西的时候记得带伞。", + "地得工工作很努力。", +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokenizer-repo", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + import logging + logging.disable(logging.CRITICAL) + sys.path.insert(0, str(args.tokenizer_repo.resolve())) + from zipvoice.tokenizer.tokenizer import EmiliaTokenizer + + tokenizer = EmiliaTokenizer(token_file=str(args.model_dir / "tokens.txt")) + out = {} + for text in CORPUS: + ids = tokenizer.texts_to_token_ids([text])[0] + tokens = tokenizer.texts_to_tokens([text])[0] + out[text] = {"ids": ids, "tokens": tokens} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8") + print(f"wrote {args.output} ({len(out)} cases)") + + +if __name__ == "__main__": + main() diff --git a/tests/zipvoice/zipvoice_parity_main.cpp b/tests/zipvoice/zipvoice_parity_main.cpp new file mode 100644 index 000000000..aeaddacc3 --- /dev/null +++ b/tests/zipvoice/zipvoice_parity_main.cpp @@ -0,0 +1,408 @@ +// ZipVoice parity harness against the moonlight fixture produced by the +// zipvoice-lite MLX port (zipvoice/mlx/fixtures/moonlight_distill_case.npz). +// The fixture stores the PyTorch reference inputs and outputs for the +// ZipVoice-Distill checkpoint: +// tokens, prompt_tokens, prompt_features(+lens), x0, x1 (full sample), +// text_condition (text encoder + duration prediction output). +// Usage: zipvoice_parity +#include "engine/community_models/zipvoice/synthesize.h" +#include "engine/framework/audio/wav_writer.h" +#include "engine/framework/model_spec/package.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// ---- minimal ZIP (STORED) + .npy reader ----------------------------------- + +struct NpyArray { + std::string name; + std::vector shape; + char typechar = 'f'; // 'f' float32, 'i' int32 + std::vector data; +}; + +uint16_t rd16(const uint8_t * p) { return uint16_t(p[0]) | (uint16_t(p[1]) << 8); } +uint32_t rd32(const uint8_t * p) { + return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24); +} + +bool parse_npy(const uint8_t * raw, size_t size, NpyArray & arr) { + if (size < 10 || std::memcmp(raw, "\x93NUMPY", 6) != 0) { + std::fprintf(stderr, "bad npy magic\n"); + std::exit(2); + } + const uint16_t header_len = rd16(raw + 8); + const std::string header(reinterpret_cast(raw + 10), header_len); + const size_t data_offset = 10 + header_len; + // 'descr': '(header[p]))) { + int64_t v = 0; + while (std::isdigit(static_cast(header[p]))) { + v = v * 10 + (header[p] - '0'); + ++p; + } + arr.shape.push_back(v); + } else { + ++p; + } + } + arr.data.assign(raw + data_offset, raw + size); + return true; +} + +std::vector load_npz(const std::string & path) { + std::ifstream f(path, std::ios::binary); + if (!f) { std::fprintf(stderr, "cannot open %s\n", path.c_str()); std::exit(1); } + std::vector zip((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + // find EOCD + size_t eocd = zip.size() - 22; + while (eocd > 0 && rd32(&zip[eocd]) != 0x06054b50) --eocd; + if (rd32(&zip[eocd]) != 0x06054b50) { std::fprintf(stderr, "bad npz\n"); std::exit(2); } + const uint16_t entries = rd16(&zip[eocd + 10]); + const uint32_t cd_offset = rd32(&zip[eocd + 16]); + // collect (name, local offset) in directory order + struct Entry { std::string name; uint32_t local; }; + std::vector directory; + uint32_t off = cd_offset; + for (uint16_t i = 0; i < entries; ++i) { + if (rd32(&zip[off]) != 0x02014b50) break; + const uint16_t name_len = rd16(&zip[off + 28]); + const uint16_t extra_len = rd16(&zip[off + 30]); + const uint16_t comment_len = rd16(&zip[off + 32]); + const uint32_t local = rd32(&zip[off + 42]); + directory.push_back({std::string(reinterpret_cast(&zip[off + 46]), name_len), local}); + off += 46 + name_len + extra_len + comment_len; + } + std::vector arrays; + for (size_t i = 0; i < directory.size(); ++i) { + const uint32_t local = directory[i].local; + const std::string & name = directory[i].name; + uint32_t csize = rd32(&zip[local + 18]); + const size_t data_start = local + 30 + rd16(&zip[local + 26]) + rd16(&zip[local + 28]); + if (csize == 0xFFFFFFFFu) { + // ZIP64 sentinel: derive the size from the neighbouring entry + // (STORED entries are laid out contiguously). + const size_t next = (i + 1 < directory.size()) ? directory[i + 1].local : cd_offset; + if (next < data_start) { std::fprintf(stderr, "bad npz entry %s\n", name.c_str()); std::exit(2); } + csize = static_cast(next - data_start); + } + if (name.size() > 4 && name.substr(name.size() - 4) == ".npy") { + NpyArray arr; + if (parse_npy(&zip[data_start], csize, arr)) { + arr.name = name.substr(0, name.size() - 4); + arrays.push_back(std::move(arr)); + } + } + } + return arrays; +} + +double cosine(const std::vector & a, const std::vector & b) { + double dot = 0, na = 0, nb = 0; + const size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) { + dot += double(a[i]) * double(b[i]); + na += double(a[i]) * double(a[i]); + nb += double(b[i]) * double(b[i]); + } + return dot / (std::sqrt(na) * std::sqrt(nb) + 1e-12); +} + +double max_abs_diff(const std::vector & a, const std::vector & b) { + double m = 0; + const size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) m = std::max(m, double(std::fabs(a[i] - b[i]))); + return m; +} + +bool check(const std::string & name, const std::vector & got, + const std::vector & want, float atol, float rtol) { + bool ok = !got.empty() && got.size() == want.size(); + double squared = 0; + for (size_t i = 0; i < std::min(got.size(), want.size()); ++i) { + const double diff = std::abs(double(got[i]) - want[i]); + squared += diff * diff; + ok = ok && std::isfinite(got[i]) && std::isfinite(want[i]) && + diff <= atol + rtol * std::abs(want[i]); + } + std::printf("%-24s size=%zu/%zu cosine=%.9f maxdiff=%.6g rmse=%.6g %s\n", + name.c_str(), got.size(), want.size(), cosine(got, want), + max_abs_diff(got, want), std::sqrt(squared / std::max(1, got.size())), + ok ? "OK" : "FAIL"); + return ok; +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s [taps.npz] [vocos-path] [output.wav]\n", argv[0]); + return 2; + } + const auto arrays = load_npz(argv[1]); + // Arrays written by zipvoice/mlx/fixtures/build_fixture.py. Keyed by name: + // tokens, prompt_tokens, prompt_features(+lens), text_condition, x0, x1. + const auto find_arr = [&](const char * key) -> const NpyArray & { + for (const auto & a : arrays) { + if (a.name == key) return a; + } + std::fprintf(stderr, "fixture missing array '%s'\n", key); + std::exit(2); + }; + const auto & tokens_arr = find_arr("tokens"); + const auto & prompt_tokens_arr = find_arr("prompt_tokens"); + const auto & prompt_features = find_arr("prompt_features"); + const auto & prompt_features_lens = find_arr("prompt_features_lens"); + const auto & text_condition = find_arr("text_condition"); + const auto & x0 = find_arr("x0"); + const auto & x1 = find_arr("x1"); + + std::vector tokens(tokens_arr.data.size() / 4); + std::memcpy(tokens.data(), tokens_arr.data.data(), tokens_arr.data.size()); + std::vector prompt_tokens(prompt_tokens_arr.data.size() / 4); + std::memcpy(prompt_tokens.data(), prompt_tokens_arr.data.data(), prompt_tokens_arr.data.size()); + int64_t prompt_len = 0; + std::memcpy(&prompt_len, prompt_features_lens.data.data(), sizeof(int32_t)); + + const auto to_floats = [](const NpyArray & a) { + std::vector v(a.data.size() / 4); + std::memcpy(v.data(), a.data.data(), a.data.size()); + return v; + }; + + const std::string model = argv[2]; + std::optional bundle; + if (std::filesystem::path(model).extension() == ".gguf") { + bundle = engine::model_spec::load_resource_bundle_for_family(model, "zipvoice"); + } + const auto * resources = bundle ? &*bundle : nullptr; + bool checks_ok = true; + engine::models::zipvoice::ZipVoiceComputeDevice device; + device.backend_type = engine::core::BackendType::Cpu; // parity: bit-stable reference path + device.threads = 8; + + // ---- stage 0: raw text encoder vs torch taps (when provided) ---- + if (argc > 3) { + const auto taps = load_npz(argv[3]); + const auto find_tap_opt = [&](const char * key) -> const NpyArray * { + for (const auto & a : taps) { + if (a.name == key) return &a; + } + return nullptr; + }; + // torch embed is [1, S+1, feat]; row-major == [S+1, F] rows (optional: + // the layer0 tap file lacks it) + const NpyArray * embed_ref = find_tap_opt("embed"); + // reference runs the encoder on prompt_tokens + tokens (pad appended + // inside the hook) + std::vector cat_ids(prompt_tokens); + cat_ids.insert(cat_ids.end(), tokens.begin(), tokens.end()); + engine::models::zipvoice::ZipVoiceLayerTaps got_taps; + const auto got_embed = engine::models::zipvoice::zipvoice_text_encoder_raw( + model, cat_ids, &got_taps, device, resources); + if (embed_ref != nullptr) { + const auto want_embed = to_floats(*embed_ref); + checks_ok &= check("text_encoder_raw", got_embed, want_embed, 2e-6F, 2e-5F); + } + // per-layer outputs (torch (S+1, 1, C) rows) for bisecting + for (size_t li = 0; li < got_taps.layers.size(); ++li) { + const std::string name = "layer" + std::to_string(li); + const NpyArray * ref = find_tap_opt(name.c_str()); + if (ref == nullptr) continue; + const auto want = to_floats(*ref); + const auto & got = got_taps.layers[li]; + checks_ok &= check(name, got, want, 3e-5F, 2e-5F); + } + const size_t stage_count = got_taps.stages.size(); + const char * got_names[] = { + "layer_in", "attn_qtb", "attn_ktb", "attn_inproj", "attn_scores", + "attn_pos", "attn_w", "ff1_inproj", "ff1_act", "ff1", "na", + "na_gated", "na_gtb", "na_attnrep", "na_attended", "na_mul", "na_y2", "na_h", "sa1", + "conv1", "ff2", "bypass_mid", "sa2", "conv2", "ff3", "norm"}; + if (stage_count != sizeof(got_names) / sizeof(got_names[0])) { + throw std::runtime_error("text stage tap/name count mismatch"); + } + for (size_t si = 0; si < stage_count; ++si) { + const auto & got = got_taps.stages[si]; + const char * ref_key = nullptr; + // map got -> ref by semantic name + const char * gn = got_names[si]; + if (std::strcmp(gn, "layer_in") == 0) ref_key = "layer_in"; + else if (std::strcmp(gn, "attn_qtb") == 0) ref_key = "attn_qtb_ref"; + else if (std::strcmp(gn, "attn_ktb") == 0) ref_key = "attn_ktb_ref"; + else if (std::strcmp(gn, "attn_inproj") == 0) ref_key = "attn_inproj_ref"; + else if (std::strcmp(gn, "attn_scores") == 0) ref_key = "attn_scores_ref"; + else if (std::strcmp(gn, "attn_pos") == 0) ref_key = "attn_pos_ref"; + else if (std::strcmp(gn, "attn_w") == 0) ref_key = "self_attn_weights"; + else if (std::strcmp(gn, "ff1_inproj") == 0) ref_key = "ff1_inproj"; + else if (std::strcmp(gn, "ff1_act") == 0) ref_key = "ff1_act"; + else if (std::strcmp(gn, "ff1") == 0) ref_key = "feed_forward1"; + else if (std::strcmp(gn, "na") == 0) ref_key = "nonlin_attention"; + else if (std::strcmp(gn, "na_gated") == 0) ref_key = "na_gated_ref"; + else if (std::strcmp(gn, "na_gtb") == 0) ref_key = "na_gtb_ref"; + else if (std::strcmp(gn, "na_mul") == 0) ref_key = "na_mul_ref"; + else if (std::strcmp(gn, "na_y2") == 0) ref_key = "na_y2_ref"; + else if (std::strcmp(gn, "na_h") == 0) ref_key = "na_h_ref"; + else if (std::strcmp(gn, "na_attnrep") == 0) ref_key = "self_attn_weights"; + else if (std::strcmp(gn, "na_attended") == 0) ref_key = "na_attended_ref"; + else if (std::strcmp(gn, "sa1") == 0) ref_key = "self_attn1"; + else if (std::strcmp(gn, "conv1") == 0) ref_key = "conv_module1"; + else if (std::strcmp(gn, "ff2") == 0) ref_key = "feed_forward2"; + else if (std::strcmp(gn, "bypass_mid") == 0) ref_key = "bypass_mid"; + else if (std::strcmp(gn, "sa2") == 0) ref_key = "self_attn2"; + else if (std::strcmp(gn, "conv2") == 0) ref_key = "conv_module2"; + else if (std::strcmp(gn, "ff3") == 0) ref_key = "feed_forward3"; + else if (std::strcmp(gn, "norm") == 0) ref_key = "norm"; + if (ref_key == nullptr) continue; + // torch attention weights are (H, B, T, T); got is [S*T? ] handled below + if (getenv("ZV_DUMP") != nullptr) { + const std::string path = std::string("/tmp/zv_mine_") + gn + ".bin"; + std::ofstream f(path, std::ios::binary); + f.write(reinterpret_cast(got.data()), + static_cast(got.size() * sizeof(float))); + } + const NpyArray * ref = find_tap_opt(ref_key); + if (ref == nullptr) continue; + auto want = to_floats(*ref); + if ((std::strcmp(gn, "attn_qtb") == 0 || std::strcmp(gn, "attn_ktb") == 0) && ref->shape.size() == 3) { + const int64_t D = ref->shape[0], T = ref->shape[1], H = ref->shape[2]; + auto reordered = want; + for (int64_t d = 0; d < D; ++d) + for (int64_t t = 0; t < T; ++t) + for (int64_t h = 0; h < H; ++h) + reordered[d + D * (t + T * h)] = want[h + H * (t + T * d)]; + want = std::move(reordered); + } + if (std::strcmp(gn, "na_attended") == 0 && ref->shape.size() == 4) { + const int64_t H = ref->shape[0], B = ref->shape[1], T = ref->shape[2], D = ref->shape[3]; + auto reordered = want; + for (int64_t h = 0; h < H; ++h) + for (int64_t b = 0; b < B; ++b) + for (int64_t t = 0; t < T; ++t) + for (int64_t d = 0; d < D; ++d) + reordered[t + T * (d + D * (h + H * b))] = want[d + D * (t + T * (b + B * h))]; + want = std::move(reordered); + } + if (std::strcmp(gn, "na_attnrep") == 0 && ref->shape.size() == 4 && ref->shape[1] == 1) { + want.resize(static_cast(ref->shape[2] * ref->shape[3])); + } + // attention weights need a 4D reorder: torch (H, B, tgt, src) + // row-major vs ggml [src (ne0), tgt, H, B] + if (std::strcmp(gn, "attn_w") == 0) { + const int64_t H = ref->shape[0], B = ref->shape[1]; + const int64_t TT = ref->shape[2], TS = ref->shape[3]; + std::vector reordered(want.size(), 0.0F); + for (int64_t h = 0; h < H; ++h) + for (int64_t b = 0; b < B; ++b) + for (int64_t tt = 0; tt < TT; ++tt) + for (int64_t ts = 0; ts < TS; ++ts) { + const size_t torch_flat = static_cast( + h * B * TT * TS + b * TT * TS + tt * TS + ts); + const size_t mine_flat = static_cast( + ts + tt * TS + h * TS * TT + b * TS * TT * H); + reordered[mine_flat] = want[torch_flat]; + } + want = std::move(reordered); + } + if (got.size() != want.size()) { + std::printf(" stage %-18s: size cpp=%zu ref=%zu SKIP\n", gn, got.size(), want.size()); + checks_ok = false; + continue; + } + checks_ok &= check(gn, got, want, 3e-5F, 2e-5F); + + } + } + + // ---- stage 1: text_condition ---- + const auto want_text = to_floats(text_condition); + const auto got_text = engine::models::zipvoice::zipvoice_text_condition( + model, tokens, prompt_tokens, prompt_len, 1.0F, device, resources); + std::printf("text_condition: cpp=%zu ref=%zu cosine=%.6f maxdiff=%.4g\n", + got_text.size(), want_text.size(), + cosine(got_text, want_text), max_abs_diff(got_text, want_text)); + + // ---- stage 2: full sample (distill, 8 steps, guidance 3.0, t_shift 0.5) ---- + const auto prompt_feats = to_floats(prompt_features); + const auto want_x1 = to_floats(x1); + const auto got_x1 = engine::models::zipvoice::zipvoice_sample( + model, tokens, prompt_tokens, prompt_feats, prompt_len, + to_floats(x0), 8, 3.0F, 0.5F, 1.0F, device, resources); + std::printf("x1: cpp=%zu ref=%zu cosine=%.6f maxdiff=%.4g\n", + got_x1.size(), want_x1.size(), + cosine(got_x1, want_x1), max_abs_diff(got_x1, want_x1)); + + const bool text_ok = check("text_condition", got_text, want_text, 2e-6F, 2e-5F); + const bool x1_ok = check("sample", got_x1, want_x1, 3e-4F, 2e-4F); + for (const auto & array : arrays) { + if (array.name.rfind("velocity_", 0) != 0) continue; + int length = 0, step = 0; + if (std::sscanf(array.name.c_str(), "velocity_%d_%d", &length, &step) != 2) return 2; + auto noise = to_floats(x0); + auto text = want_text; + auto speech = to_floats(find_arr("speech_condition")); + noise.resize(length * 100); + text.resize(length * 100); + speech.resize(length * 100); + const auto velocity = engine::models::zipvoice::zipvoice_velocity( + model, noise, text, speech, length, step == 0 ? 0.0F : 0.6F, 3.0F, device, 1, resources); + checks_ok &= check(array.name, velocity, to_floats(array), 3e-4F, 2e-4F); + } + for (const auto & array : arrays) { + if (array.name == "batch_velocity") { + checks_ok &= check("batch_velocity", engine::models::zipvoice::zipvoice_velocity( + model, to_floats(find_arr("batch_x")), to_floats(find_arr("batch_text")), + to_floats(find_arr("batch_speech")), array.shape[1], 0.6F, 3.0F, device, 2, resources), + to_floats(array), 3e-4F, 2e-4F); + } + if (array.name == "fbank_mel") { + checks_ok &= check("fbank", engine::models::zipvoice::zipvoice_logmel( + to_floats(find_arr("fbank_wav"))), to_floats(array), 5e-5F, 2e-5F); + } + if (array.name == "vocos_audio") { + const std::string vocos = argc > 4 ? argv[4] : model; + checks_ok &= check("vocos", engine::models::zipvoice::zipvoice_vocos_decode( + vocos, to_floats(find_arr("vocos_mel"))), to_floats(array), 2e-4F, 2e-3F); + } + } + std::printf("%s %s\n", text_ok ? "TEXT_CONDITION OK" : "TEXT_CONDITION FAIL", + x1_ok ? "SAMPLE OK" : "SAMPLE FAIL"); + if (argc > 5 && text_ok && x1_ok && checks_ok) { + std::vector mel(got_x1.begin() + prompt_len * 100, got_x1.end()); + for (float & value : mel) value /= 0.1F; + auto audio = engine::models::zipvoice::zipvoice_vocos_decode(argv[4], mel); + double energy = 0; + for (float value : audio) { + if (!std::isfinite(value)) throw std::runtime_error("non-finite audio sample"); + energy += double(value) * value; + } + if (audio.empty() || energy / audio.size() < 1e-8) throw std::runtime_error("silent synthesis output"); + engine::audio::write_pcm16_wav(argv[5], 24000, 1, audio); + std::printf("Wrote %s: %.3f seconds, RMS %.6f\n", argv[5], audio.size() / 24000.0, + std::sqrt(energy / audio.size())); + } + return (text_ok && x1_ok && checks_ok) ? 0 : 1; +} diff --git a/tests/zipvoice/zipvoice_zh_tokens_main.cpp b/tests/zipvoice/zipvoice_zh_tokens_main.cpp new file mode 100644 index 000000000..ea2956c2e --- /dev/null +++ b/tests/zipvoice/zipvoice_zh_tokens_main.cpp @@ -0,0 +1,107 @@ +// Token-level parity test for the zipvoice Chinese frontend (emilia mode): +// compares EmiliaTokenizer::encode against reference ids dumped from the +// upstream tokenizer by tests/zipvoice/zh_reference_ids.py. +// +// Usage: zipvoice_zh_tokens +#include "engine/community_models/zipvoice/emilia_tokenizer.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::string slurp(const std::string & path) { + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error("cannot open " + path); + return std::string(std::istreambuf_iterator(in), std::istreambuf_iterator()); +} + +// Minimal JSON parsing for the flat {"text": {"ids": [...], "tokens": [...]}} +// file (corpus keys contain no escaped quotes). +struct Case { std::string text; std::vector ids; }; + +std::vector parse_cases(const std::string & json) { + std::vector cases; + size_t pos = 0; + while (true) { + const auto ids_key = json.find("\"ids\"", pos); + if (ids_key == std::string::npos) break; + const auto colon = json.rfind(':', ids_key); + const auto key_end = json.rfind('"', colon); + const auto key_start = json.rfind('"', key_end - 1); + if (colon == std::string::npos || key_end == std::string::npos || + key_start == std::string::npos) break; + Case entry; + entry.text = json.substr(key_start + 1, key_end - key_start - 1); + const auto ids_open = json.find('[', ids_key); + const auto ids_close = json.find(']', ids_open); + std::string ids_raw = json.substr(ids_open + 1, ids_close - ids_open - 1); + size_t start = 0; + while (start < ids_raw.size()) { + const auto comma = ids_raw.find(',', start); + const auto piece = ids_raw.substr(start, + comma == std::string::npos ? std::string::npos : comma - start); + if (!piece.empty()) entry.ids.push_back(std::stoi(piece)); + if (comma == std::string::npos) break; + start = comma + 1; + } + cases.push_back(std::move(entry)); + pos = ids_close; + } + return cases; +} + +std::unordered_map load_vocab(const std::filesystem::path & dir) { + std::ifstream in(dir / "tokens.txt"); + if (!in) throw std::runtime_error("cannot open tokens.txt in " + dir.string()); + std::unordered_map vocab; + std::string line; + while (std::getline(in, line)) { + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + vocab.emplace(line.substr(0, tab), std::stoi(line.substr(tab + 1))); + } + return vocab; +} + +} // namespace + +int main(int argc, char ** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + using namespace engine::models::zipvoice; + const std::filesystem::path model_dir = argv[2]; + const auto vocab = load_vocab(model_dir); + EmiliaTokenizer tokenizer( + EmiliaTokenizer::TablePaths::from_model_dir(model_dir), vocab, + EmiliaTokenizer::EspeakConfig{ + "/opt/homebrew/lib/libespeak-ng.dylib", "", "en-us"}); + + const auto cases = parse_cases(slurp(argv[1])); + int failures = 0; + for (const auto & c : cases) { + const auto got = tokenizer.encode(c.text); + bool ok = got.size() == c.ids.size(); + for (size_t i = 0; ok && i < got.size(); ++i) ok = got[i] == c.ids[i]; + if (!ok) { + ++failures; + std::printf("FAIL %-30s got=%zu want=%zu\n", c.text.c_str(), got.size(), c.ids.size()); + std::printf(" got :"); + for (int32_t v : got) std::printf(" %d", v); + std::printf("\n want:"); + for (int32_t v : c.ids) std::printf(" %d", v); + std::printf("\n"); + } else { + std::printf("OK %s (%zu ids)\n", c.text.c_str(), got.size()); + } + } + std::printf("%d/%zu cases passed\n", static_cast(cases.size() - failures), cases.size()); + return failures == 0 ? 0 : 1; +} diff --git a/tools/community_models/convert_zipvoice.py b/tools/community_models/convert_zipvoice.py new file mode 100644 index 000000000..cfb372509 --- /dev/null +++ b/tools/community_models/convert_zipvoice.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Convert ZipVoice / ZipVoice-Distill torch checkpoints to audio.cpp packages. + +ZipVoice (https://github.com/k2-fsa/ZipVoice, k2-fsa/ZipVoice on HuggingFace) +ships `model.pt` checkpoints (a pickled dict with a "model" state dict), a +`model.json` architecture config, and a `tokens.txt` phone vocab. audio.cpp +cannot read the nested pickle directly, so this script first flattens the +state dict to safetensors (the development format) and then packages it into +a self-contained GGUF with two tensor namespaces: + + model.* — the ZipVoice flow-matching model (raw torch names) + vocos.* — the Vocos mel-24kHz vocoder used to decode generated features + +`tokens.txt` and `model.json` are copied next to the output so the GGUF plus +those two files form a complete model directory for +`audiocpp_cli --family zipvoice`. + +Examples: + # ZipVoice-Distill (8 steps, guidance-scale embedding) with bundled vocoder + python3 tools/community_models/convert_zipvoice.py \ + --model-dir /models/ZipVoice/zipvoice_distill \ + --vocos /models/vocos-mel-24khz/vocos.safetensors \ + --converter build/bin/audiocpp_gguf + + # safetensors only (development format, no GGUF step) + python3 tools/community_models/convert_zipvoice.py \ + --model-dir /models/ZipVoice/zipvoice_distill --safetensors-only +""" +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import torch + + +def load_state_dict(model_dir: Path, checkpoint_name: str) -> dict: + ckpt_path = model_dir / checkpoint_name + if not ckpt_path.is_file(): + raise SystemExit(f"checkpoint not found: {ckpt_path}") + ckpt = torch.load(str(ckpt_path), map_location="cpu", weights_only=False) + if isinstance(ckpt, dict) and "model" in ckpt: + state = ckpt["model"] + else: + state = ckpt + if not isinstance(state, dict): + raise SystemExit(f"unexpected checkpoint layout in {ckpt_path}") + return {name: tensor.contiguous().to(torch.float32) + for name, tensor in state.items() + if isinstance(tensor, torch.Tensor)} + + +def write_safetensors(state: dict, output: Path) -> None: + from safetensors.torch import save_file + output.parent.mkdir(parents=True, exist_ok=True) + save_file(state, str(output)) + print(f"wrote {output} ({output.stat().st_size / 1e6:.1f} MB, " + f"{len(state)} tensors)") + + +def convert_gguf(converter: Path, safetensors: Path, vocos: Path, + output: Path, quant_type: str, overwrite: bool, + root: Path) -> None: + command = [ + str(converter), + "--input", f"model={safetensors}", + "--input", f"vocos={vocos}", + "--root", str(root), + "--family", "zipvoice", + "--type", quant_type, + "--output", str(output), + ] + if overwrite: + command.append("--overwrite") + print("+", " ".join(command)) + subprocess.run(command, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--model-dir", type=Path, required=True, + help="ZipVoice model dir with model.pt / model.json / tokens.txt") + parser.add_argument("--checkpoint-name", default="model.pt") + parser.add_argument("--vocos", type=Path, + default=Path("/models/vocos-mel-24khz/vocos.safetensors"), + help="vocos.safetensors (bundled into the GGUF)") + parser.add_argument("--converter", type=Path, + default=Path("build/bin/audiocpp_gguf")) + parser.add_argument("--output-dir", type=Path, default=None, + help="default: /converted") + parser.add_argument("--type", default="orig", + choices=["orig", "f16", "bf16", "q8_0", "q2_k", "q3_k", + "q4_k", "q5_k", "q6_k"], + help="GGUF storage type (default orig = keep f32)") + parser.add_argument("--name", default=None, + help="package base name (default: model-dir name, lowercased)") + parser.add_argument("--safetensors-only", action="store_true", + help="stop after writing the flattened safetensors") + parser.add_argument("--no-vocos", action="store_true", + help="do not bundle a vocoder namespace into the GGUF") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + model_dir = args.model_dir.resolve() + for required in ("model.json", "tokens.txt"): + if not (model_dir / required).is_file(): + raise SystemExit(f"{required} missing in {model_dir}") + with (model_dir / "model.json").open() as f: + config = json.load(f) + has_guidance_embed = ( + config.get("distill", {}).get("guidance_scale_embed", False) + if isinstance(config.get("distill"), dict) else False + ) + name = args.name or model_dir.name.lower().replace("_", "-") + out_dir = (args.output_dir or model_dir / "converted").resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + state = load_state_dict(model_dir, args.checkpoint_name) + print(f"loaded {len(state)} tensors from {model_dir / args.checkpoint_name}") + + # Fixed name matching model_specs/zipvoice.json sources[format=safetensors] + # ("model:zipvoice-orig.safetensors"); the GGUF keeps the - naming. + safetensors_path = out_dir / "zipvoice-orig.safetensors" + write_safetensors(state, safetensors_path) + + # dev-format directory: safetensors + config + vocab + shutil.copyfile(model_dir / "tokens.txt", out_dir / "tokens.txt") + shutil.copyfile(model_dir / "model.json", out_dir / "model.json") + # Chinese frontend sidecars (pypinyin tables + jieba dictionaries), when + # the model directory was staged by export_zipvoice_zh_dict.py. Staged + # into out_dir BEFORE packaging: the converter root is scanned for + # sidecars, so these ride inside the GGUF and a converted package is a + # single self-sufficient file (loose copies below remain for the + # directory layout). + for sidecar in sorted(model_dir.glob("zh_*")): + shutil.copyfile(sidecar, out_dir / sidecar.name) + staged_zh = sorted(p.name for p in out_dir.glob("zh_*")) + print(f"copied tokens.txt / model.json -> {out_dir}" + + (f" + {len(staged_zh)} zh frontend sidecars" if staged_zh else "")) + + if args.safetensors_only: + return + + converter = args.converter.resolve() + if not converter.is_file(): + raise SystemExit(f"converter not found: {converter} (build target audiocpp_gguf)") + + gguf_dir = out_dir / name + gguf_dir.mkdir(parents=True, exist_ok=True) + gguf_path = gguf_dir / f"{name}-{args.type}.gguf" + # NOTE: --root must point at a directory with real (non-symlinked) + # sidecar files; HF-cache snapshots use blob symlinks that the sidecar + # scanner does not enumerate, so pass the flattened dev directory that + # already carries tokens.txt / model.json copies. + root = out_dir if (out_dir / "tokens.txt").is_file() else model_dir + if args.no_vocos: + command = [ + str(converter), + "--input", f"model={safetensors_path}", + "--root", str(root), + "--family", "zipvoice", + "--type", args.type, + "--output", str(gguf_path), + ] + if args.overwrite: + command.append("--overwrite") + print("+", " ".join(command)) + subprocess.run(command, check=True) + else: + vocos = args.vocos.resolve() + if not vocos.is_file(): + raise SystemExit(f"vocos checkpoint not found: {vocos} " + "(pass --no-vocos or --vocos )") + convert_gguf(converter, safetensors_path, vocos, gguf_path, + args.type, args.overwrite, root) + shutil.copyfile(model_dir / "tokens.txt", gguf_dir / "tokens.txt") + shutil.copyfile(model_dir / "model.json", gguf_dir / "model.json") + # Chinese frontend sidecars (pypinyin tables + jieba dictionaries), when + # the model directory was staged by export_zipvoice_zh_dict.py. + for sidecar in sorted(model_dir.glob("zh_*")): + shutil.copyfile(sidecar, gguf_dir / sidecar.name) + print(f"package ready: {gguf_path}") + + +if __name__ == "__main__": + main() diff --git a/tools/community_models/export_zipvoice_zh_dict.py b/tools/community_models/export_zipvoice_zh_dict.py new file mode 100644 index 000000000..58e072ae0 --- /dev/null +++ b/tools/community_models/export_zipvoice_zh_dict.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Export Chinese G2P dictionaries for the ZipVoice audio.cpp frontend. + +The upstream ZipVoice EmiliaTokenizer converts Chinese text with +jieba + pypinyin (TONE3 style, tone sandhi, neutral tone = 5) and splits +every syllable into an initial token (e.g. "w0") plus a final token +(e.g. "o3"). audio.cpp implements the lookup in C++ but must reproduce +pypinyin's syllable splitting exactly, so this script bakes the split +into the exported tables: + + zh_chars.tsv \\t (default reading) + zh_phrases.tsv \\t (raw tones, no sandhi) + zh_syllables.tsv \\t (for tags) + +Tone sandhi (3-3 runs, 一/不) is applied by the C++ frontend at run time, +mirroring lazy_pinyin(tone_sandhi=True); the tables therefore stay raw. + +This script also stages the jieba segmentation dictionaries used by the +C++ frontend (the model-local JiebaSegmenter, equivalent to python +jieba.cut with HMM on): + + zh_jieba_dict.txt lines (jieba jieba.dict.utf8) + zh_hmm_model.txt HMM start/trans/emit tables (jieba hmm_model.utf8) + +The dictionaries are not vendored in this repository; they are downloaded +from the cppjieba mirror commit pinned below (SHA-256 verified, cached +under ~/.cache/audiocpp) so the packaged model is self-contained. Pass +--cppjieba-dict to copy from a local cppjieba checkout instead +(offline escape hatch). + +Splitting follows pypinyin's to_initials(strict=False) + +to_finals_tone3(strict=False, neutral_tone_with_five=True): y/w ARE +initials ("yi3" -> "y0 i3"), and ju/qu/xu keep "u" (only nv/lv give "v"). + +Run with the same Python environment used for the upstream checkout, e.g. + /path/to/ZipVoice/.venv/bin/python \\ + tools/community_models/export_zipvoice_zh_dict.py \\ + --output-dir /models/ZipVoice/zipvoice_distill +""" +import argparse +import hashlib +import unicodedata +import urllib.request +from pathlib import Path + +# These jieba dictionaries are pinned to a cppjieba mirror commit (cppjieba +# redistributes the jieba dictionaries); byte-for-byte identity is enforced by +# SHA-256 so exported packages stay reproducible. +CPPJIEBA_COMMIT = "8f171de" +CPPJIEBA_DICT_URL = ("https://raw.githubusercontent.com/yanyiwu/cppjieba/" + + CPPJIEBA_COMMIT + "/dict/{name}") +CPPJIEBA_DICT_SHA256 = { + "jieba.dict.utf8": + "6f7d4350e8861ef4139b2e3a6fad05430c19ae71f4b8378190edecac8aae2e6a", + "hmm_model.utf8": + "f17790586ac86dd048c8adffed052c4bd2b28ed0682972c1275e59040c0589a7", +} +DEFAULT_DICT_CACHE_DIR = Path.home() / ".cache" / "audiocpp" / "cppjieba-dict" + +_TONE_MARKS = { + "ā": ("a", 1), "á": ("a", 2), "ǎ": ("a", 3), "à": ("a", 4), + "ē": ("e", 1), "é": ("e", 2), "ě": ("e", 3), "è": ("e", 4), + "ī": ("i", 1), "í": ("i", 2), "ǐ": ("i", 3), "ì": ("i", 4), + "ō": ("o", 1), "ó": ("o", 2), "ǒ": ("o", 3), "ò": ("o", 4), + "ū": ("u", 1), "ú": ("u", 2), "ǔ": ("u", 3), "ù": ("u", 4), + "ǖ": ("v", 1), "ǘ": ("v", 2), "ǚ": ("v", 3), "ǜ": ("v", 4), + "ń": ("n", 2), "ň": ("n", 3), "ǹ": ("n", 4), + "ḿ": ("m", 2), +} + +_INITIALS = [ + "zh", "ch", "sh", "b", "p", "m", "f", "d", "t", "n", "l", "g", "k", + "h", "j", "q", "x", "r", "z", "c", "s", "y", "w", +] + + +def to_tone3(syllable: str) -> str: + """Convert an accented pinyin syllable (e.g. "xiǎo") to TONE3 ("xiao3").""" + syllable = unicodedata.normalize("NFKC", syllable) + out = [] + tone = 0 + i = 0 + while i < len(syllable): + ch = syllable[i] + two = syllable[i:i + 2] + if two in _TONE_MARKS: + base, t = _TONE_MARKS[two] + out.append(base) + tone = t + i += 2 + continue + if ch in _TONE_MARKS: + base, t = _TONE_MARKS[ch] + out.append(base) + tone = t + else: + out.append(ch) + i += 1 + if tone == 0: + tone = 5 # neutral-tone syllables carry no mark + return "".join(out) + str(tone) + + +def split_syllable(tone3: str) -> list[str]: + """Split a TONE3 syllable into initial ("X0") + final ("Ytone") tokens.""" + tone = tone3[-1] + body = tone3[:-1] + if not body.isalpha(): + return [tone3] + initial = "" + rest = body + for cand in _INITIALS: + if body.startswith(cand): + initial = cand + rest = body[len(cand):] + break + tokens = [] + if initial: + tokens.append(initial + "0") + if rest: + tokens.append(rest + tone) + return tokens + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def stage_jieba_dicts(output_dir: Path, cache_dir: Path, + override_dir: Path | None) -> None: + """Write zh_jieba_dict.txt / zh_hmm_model.txt next to tokens.txt. + + Downloads the pinned upstream dictionaries into cache_dir on first use. + --cppjieba-dict overrides the download with a local cppjieba dict/ + directory (copied as-is; its hashes are printed for visibility). + """ + for name in ("jieba.dict.utf8", "hmm_model.utf8"): + target = output_dir / ("zh_hmm_model.txt" if name.endswith("hmm_model.utf8") + else "zh_jieba_dict.txt") + if override_dir is not None: + source = override_dir / name + if not source.is_file(): + raise SystemExit(f"--cppjieba-dict override missing {name}: {source}") + data = source.read_bytes() + note = f"copied from {source} (sha256 {_sha256(data)})" + else: + cache = cache_dir / name + if cache.is_file() and _sha256(cache.read_bytes()) == CPPJIEBA_DICT_SHA256[name]: + data = cache.read_bytes() + note = f"cached {cache}" + else: + url = CPPJIEBA_DICT_URL.format(name=name) + print(f"downloading {url}") + with urllib.request.urlopen(url, timeout=120) as response: + data = response.read() + digest = _sha256(data) + if digest != CPPJIEBA_DICT_SHA256[name]: + raise SystemExit( + f"downloaded {name} SHA-256 {digest} does not match the pinned " + f"{CPPJIEBA_DICT_SHA256[name]}; pass --cppjieba-dict to copy the " + "dictionaries from a local cppjieba checkout instead") + cache_dir.mkdir(parents=True, exist_ok=True) + cache.write_bytes(data) + note = f"downloaded (pinned {CPPJIEBA_COMMIT}, SHA-256 verified)" + target.write_bytes(data) + print(f"staged {name} -> {target} [{note}]") + + +def export_pinyin_tables(output_dir: Path) -> None: + # Deferred import: the dictionary staging above only needs the standard + # library, so the download path runs in any Python; only the table baking + # requires the upstream ZipVoice environment. + try: + from pypinyin.phrases_dict import phrases_dict + from pypinyin.pinyin_dict import pinyin_dict + except ImportError as error: + raise SystemExit( + "pypinyin is required to bake the pronunciation tables; run this " + "script with the upstream ZipVoice python environment " + f"(import failed: {error})") from error + + # Characters: pinyin_dict maps codepoint -> space-separated accented + # pinyins; the first entry is pypinyin's default reading. + bare_syllables = set() + chars_path = output_dir / "zh_chars.tsv" + with chars_path.open("w", encoding="utf-8") as f: + for codepoint, pinyins in sorted(pinyin_dict.items()): + if not (0x3400 <= codepoint <= 0x9FFF or 0xF900 <= codepoint <= 0xFAFF): + continue + # Entries are comma-separated readings (e.g. "hǔ,hù"); the + # first is pypinyin's default reading. + first = to_tone3(pinyins.split(",")[0].strip()) + bare_syllables.add(first) + tokens = split_syllable(first) + f.write(f"{chr(codepoint)}\t{' '.join(tokens)}\n") + print(f"wrote {chars_path}") + + # Phrases: raw per-syllable tones (no sandhi); longest-match lookup and + # tone sandhi happen in C++. + phrases_path = output_dir / "zh_phrases.tsv" + count = 0 + with phrases_path.open("w", encoding="utf-8") as f: + for phrase, syllables in sorted(phrases_dict.items()): + if not all(len(s) == 1 for s in syllables): + continue + tokens = [] + for entry in syllables: + tokens.extend(split_syllable(to_tone3(entry[0]))) + f.write(f"{phrase}\t{' '.join(tokens)}\n") + count += 1 + print(f"wrote {phrases_path} ({count} entries)") + + # Valid bare tone3 syllables for overrides. + syllables_path = output_dir / "zh_syllables.tsv" + with syllables_path.open("w", encoding="utf-8") as f: + for syllable in sorted(bare_syllables): + f.write(f"{syllable}\t{' '.join(split_syllable(syllable))}\n") + print(f"wrote {syllables_path} ({len(bare_syllables)} entries)") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True, + help="model directory that contains tokens.txt") + parser.add_argument("--cppjieba-dict", type=Path, default=None, + help="offline override: cppjieba dict/ directory to copy " + "from (default: download the pinned upstream " + "dictionaries, SHA-256 verified)") + parser.add_argument("--dict-cache-dir", type=Path, default=DEFAULT_DICT_CACHE_DIR, + help="download cache directory (default: %(default)s)") + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + + # jieba segmentation dictionaries for the model-local segmenter. + stage_jieba_dicts(args.output_dir, args.dict_cache_dir, args.cppjieba_dict) + + export_pinyin_tables(args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json index 8a43aa208..f463efb11 100644 --- a/webui/configs/model_params.json +++ b/webui/configs/model_params.json @@ -340,6 +340,14 @@ {"name": "duration_factor", "type": "slider", "label": "duration_factor（语速/时长倍率，>1 更慢，<1 更快）", "label_en": "duration_factor (duration multiplier; >1 slower, <1 faster)", "default": 1.0, "minimum": 0.5, "maximum": 2.0, "step": 0.05, "info": "对齐官方 IndexTTS2.5 的 duration_factor：缩放输出时长，不改变音色/内容", "info_en": "Matches official IndexTTS2.5 duration_factor: scales output duration without changing timbre or content"} ], + "zipvoice": [ + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale（引导强度，0=关闭）", "label_en": "guidance_scale (0 disables CFG)", "default": 3.0, "minimum": 0.0, "maximum": 10.0, "step": 0.5}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（Euler 步数）", "label_en": "num_inference_steps", "default": 8, "minimum": 1, "maximum": 64, "step": 1, "precision": 0}, + {"name": "t_shift", "type": "slider", "label": "t_shift（时间步偏移，越小越偏低 SNR）", "label_en": "t_shift (timestep shift)", "default": 0.5, "minimum": 0.05, "maximum": 1.0, "step": 0.05}, + {"name": "speed", "type": "slider", "label": "speed（语速倍率）", "label_en": "speed (duration multiplier)", "default": 1.0, "minimum": 0.5, "maximum": 2.0, "step": 0.05}, + {"name": "lang", "type": "text", "label": "lang（英文段 espeak 语言）", "label_en": "lang (espeak voice for English runs)", "default": "en-us", "placeholder": "en-us"} + ], + "irodori_tts": [ {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（RF 扩散步数）", "label_en": "num_inference_steps", "default": 40, "minimum": 1, "step": 1, "precision": 0}, {"name": "duration_sec", "type": "number", "label": "duration_sec（0=模型自动预测时长）", "label_en": "duration_sec (0 = auto)", "default": 0, "minimum": 0, "step": 0.5}, diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 7d557dfae..271811570 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -83,6 +83,9 @@ { "id": "echo-tts", "display_name": "Echo-TTS (voice clone)", "family": "echo_tts", "path": "models/Echo-TTS-GGUF", "task": "clon", "mode": "offline", "download_id": "echo_tts_q8_0", "min_vram_gb": 8, "input_hint_en": "**Echo-TTS**: English zero-shot cloning at 44.1 kHz. Upload a reference voice -- no transcript needed. Output is CC-BY-NC-SA and may not be used commercially." }, + { "id": "zipvoice", "display_name": "ZipVoice (中英零样本克隆)", "display_name_en": "ZipVoice (zh/en zero-shot cloning)", "family": "zipvoice", "path": "models/ZipVoice-Distill-GGUF", "task": "clon", "mode": "offline", "download_id": "zipvoice_distill_gguf", "min_vram_gb": 4, + "input_hint": "**ZipVoice**：中/英零样本声音克隆（k2-fsa TTSZipformer flow-matching + Vocos，蒸馏版 8 步采样）。上传参考音频，并在『参考文本’里填入它的逐字转写——参考音频与参考文本必须配对；合成文本支持中文、英文及混排。权重 Apache-2.0。", + "input_hint_en": "**ZipVoice**: zero-shot voice cloning in zh/en (k2-fsa TTSZipformer flow matching + Vocos, distilled 8-step sampling). Upload a reference clip and put its exact transcript in the reference-text box — the clip and the transcript must match; synthesis text supports zh/en/mixed. Weights are Apache-2.0." }, { "id": "chatterbox", "display_name": "Chatterbox (voice clone)", "family": "chatterbox", "path": "models/chatterbox", "task": "clon", "mode": "offline", "download_id": "chatterbox", "min_vram_gb": 12 }, { "id": "chatterbox-turbo", "display_name": "Chatterbox Turbo (tts)", "family": "chatterbox_turbo", "path": "models/Chatterbox-Turbo-GGUF/chatterbox-turbo-q8_0.gguf", "task": "tts", "mode": "offline", "download_id": "chatterbox_turbo_q8_0", "min_vram_gb": 4, "input_hint_en": "**Chatterbox Turbo**: fast English TTS with the built-in voice. No reference voice is required." }, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 189472bcf..23cc52eeb 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@
From 734ede99c16d0271e1a860c763687f9f46a92373 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 10:53:06 +0900 Subject: [PATCH 2/4] zipvoice: remove ggml Metal changes from PR --- .../ggml/src/ggml-metal/ggml-metal-device.cpp | 13 +++++-------- external/ggml/src/ggml-metal/ggml-metal-ops.cpp | 10 +--------- external/ggml/src/ggml-metal/ggml-metal.metal | 17 ++++------------- 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/external/ggml/src/ggml-metal/ggml-metal-device.cpp b/external/ggml/src/ggml-metal/ggml-metal-device.cpp index d1f421024..8f11f92a2 100644 --- a/external/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/external/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -752,14 +752,12 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_ return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_metal_library_t lib, const ggml_tensor * op) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_metal_library_t lib, const ggml_tensor * op) { char base[256]; char name[256]; const ggml_type tsrc0 = op->src[0]->type; - const ggml_type tsrc1 = op->src[1]->type; - const bool full_f32 = tsrc0 == GGML_TYPE_F32 && tsrc1 == GGML_TYPE_F32 && - ggml_get_op_params_i32(op, 0) == GGML_PREC_F32; + const ggml_type tsrc1 = op->src[1]->type; const bool bc_inp = op->src[0]->ne[0] % 32 != 0; @@ -778,8 +776,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_meta const int16_t r2 = (int16_t) (ne12 / op->src[0]->ne[2]); const int16_t r3 = (int16_t) (ne13 / op->src[0]->ne[3]); - snprintf(base, 256, "kernel_mul_mm_%s_%s%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1), - full_f32 ? "_prec_f32" : ""); + snprintf(base, 256, "kernel_mul_mm_%s_%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1)); snprintf(name, 256, "%s_bci=%d_bco=%d_ne12=%d_ne13=%d_r2=%d_r3=%d", base, bc_inp, bc_out, ne12, ne13, r2, r3); @@ -803,13 +800,13 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_meta res.nr0 = NRA; res.nr1 = NRB; - const size_t smem_a = NRA * N_MM_NK_TOTAL * (full_f32 ? sizeof(float) : sizeof(ggml_fp16_t)); + const size_t smem_a = NRA * N_MM_NK_TOTAL * sizeof(ggml_fp16_t); res.smem = smem_a; } else { res.nr0 = 64; res.nr1 = 32; - res.smem = full_f32 ? (8192 + 4096) : (bc_out ? 8192 : (4096 + 2048)); + res.smem = bc_out ? 8192 : (4096 + 2048); } res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y; diff --git a/external/ggml/src/ggml-metal/ggml-metal-ops.cpp b/external/ggml/src/ggml-metal/ggml-metal-ops.cpp index ab33d920d..40a5dca40 100644 --- a/external/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/external/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2371,15 +2371,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { !ggml_is_transposed(op->src[1]) && // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel - // Short F32 contractions (e.g. 32-dim QK attention) still benefit - // from tiled MM when both output axes are large enough. Keep the - // existing MV choice for small outputs and other precision modes. - props_dev->has_simdgroup_mm && - (ne00 >= 64 || (ne00 >= 32 && ne01 >= 64 && ne11 >= 32 && - op->src[0]->type == GGML_TYPE_F32 && - op->src[1]->type == GGML_TYPE_F32 && - ggml_get_op_params_i32(op, 0) == GGML_PREC_F32)) && - ne11 > ne11_mm_min) { + props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) { //GGML_LOG_INFO("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); // some Metal matrix data types require aligned pointers diff --git a/external/ggml/src/ggml-metal/ggml-metal.metal b/external/ggml/src/ggml-metal/ggml-metal.metal index 4c0f7ad39..b71b83c68 100644 --- a/external/ggml/src/ggml-metal/ggml-metal.metal +++ b/external/ggml/src/ggml-metal/ggml-metal.metal @@ -7876,15 +7876,8 @@ kernel void kernel_cpy_t_t( ushort3 ntg[[threads_per_threadgroup]]) { const int i03 = tgpig[2]; const int i02 = tgpig[1]; - const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0]; - const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; - - // The last threadgroup can contain padded rows (including ne01 == 1 - // after a permutation). Those threads must not read or overwrite the - // next channel/batch through its different source strides. - if (i01 >= args.ne01) { - return; - } + const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0]; + const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; @@ -10231,7 +10224,7 @@ kernel void kernel_mul_mm( ushort sgitg[[simdgroup_index_in_threadgroup]]) { threadgroup S0 * sa = (threadgroup S0 *)(shmem); - threadgroup S1 * sb = (threadgroup S1 *)(shmem + 64*32*sizeof(S0)); + threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096); constexpr int NR0 = 64; constexpr int NR1 = 32; @@ -10878,9 +10871,7 @@ template [[host_name("kernel_set_rows_iq4_nl_i32")]] kernel set_rows_q32_t kerne typedef decltype(kernel_mul_mm) mul_mm_t; -template [[host_name("kernel_mul_mm_f32_f32")]] kernel mul_mm_t kernel_mul_mm; -// GGML_PREC_F32 must preserve F32 operands instead of staging them as half. -template [[host_name("kernel_mul_mm_f32_f32_prec_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_f32_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; From d754903a9beccba2d20570b16b40cd945f6e81e7 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 10:53:34 +0900 Subject: [PATCH 3/4] zipvoice: register the Q8_0 model package --- docs/community_models/zipvoice.md | 8 +++++++- model_specs/zipvoice.json | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/community_models/zipvoice.md b/docs/community_models/zipvoice.md index 4e3ec11af..1411d2b6c 100644 --- a/docs/community_models/zipvoice.md +++ b/docs/community_models/zipvoice.md @@ -27,7 +27,13 @@ both timesteps), fbank, and vocos audio — all at cosine 1.0. The Chinese/Engli A ready-made, self-contained package is hosted at [davidxifeng/zipvoice-gguf](https://huggingface.co/davidxifeng/zipvoice-gguf) — the model manager downloads it directly (`zipvoice-distill-orig.gguf`, flow-matching model + bundled -Vocos + embedded frontend sidecars). To rebuild it locally: +Vocos + embedded frontend sidecars). The Q8_0 package includes the same resources: + +```bash +python3 tools/model_manager_v2.py install zipvoice_distill_q8_0 +``` + +To rebuild it locally: ```bash # 1. stage the Chinese frontend tables (requires the upstream ZipVoice python env diff --git a/model_specs/zipvoice.json b/model_specs/zipvoice.json index 4d71feca4..77e2fec5e 100644 --- a/model_specs/zipvoice.json +++ b/model_specs/zipvoice.json @@ -173,6 +173,17 @@ "files": [ "zipvoice-distill-orig.gguf" ] + }, + { + "id": "zipvoice_distill_q8_0", + "display_name": "ZipVoice-Distill GGUF (Q8_0)", + "description": "Self-contained Q8_0 GGUF with bundled Vocos and embedded text-frontend sidecars.", + "format": "gguf", + "precision": "q8_0", + "target_directory": "ZipVoice-Distill-GGUF", + "files": [ + "zipvoice-distill-q8_0.gguf" + ] } ], "dependencies": [], From 4a49141730825e8653834730d4da35bffc107377 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 11:02:49 +0900 Subject: [PATCH 4/4] zipvoice: preserve native quantized matrix weights Load ZipVoice linear and embedding weights and Vocos linear weights with Native storage. Keep convolution, normalization and bias tensors in F32 for their operators. Validated the F32 parity harness, Q8_0 CPU and Metal synthesis, and loaded tensor types for ZipVoice and all Vocos linear layers. --- src/community_models/zipvoice/weights.cpp | 4 +++- src/framework/modules/vocoders/vocos_vocoder.cpp | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/community_models/zipvoice/weights.cpp b/src/community_models/zipvoice/weights.cpp index 1104c018e..3844f2ca4 100644 --- a/src/community_models/zipvoice/weights.cpp +++ b/src/community_models/zipvoice/weights.cpp @@ -108,7 +108,9 @@ ZipVoiceWeights load_zipvoice_weights( if ((d0 >= 0 && d0 != meta.shape[1]) || (d1 >= 0 && d1 != meta.shape[0])) { throw std::runtime_error("zipvoice: " + name + " shape mismatch"); } - return weights.store->load_f32_tensor(*source, name, meta.shape); + // Linear and embedding weights can retain their GGUF quantization. + return weights.store->load_tensor( + *source, name, engine::assets::TensorStorageType::Native, meta.shape); }; const auto t1 = [&](const std::string & name, int64_t d0) { const auto meta = source->require_metadata(name); diff --git a/src/framework/modules/vocoders/vocos_vocoder.cpp b/src/framework/modules/vocoders/vocos_vocoder.cpp index 819120574..1182325cb 100644 --- a/src/framework/modules/vocoders/vocos_vocoder.cpp +++ b/src/framework/modules/vocoders/vocos_vocoder.cpp @@ -68,7 +68,11 @@ class VocosVocoder::Impl { return {f32(name + ".weight"), f32(name + ".bias")}; }; const auto linear = [&](const std::string & name) -> LinearWeights { - return {f32(name + ".weight"), f32(name + ".bias")}; + // Keep matrix weights quantized; convolution and norm weights stay F32. + const auto weight = name + ".weight"; + return {store.load_tensor(*source, weight, assets::TensorStorageType::Native, + source->require_metadata(weight).shape), + f32(name + ".bias")}; }; weights.embed = {f32("backbone.embed.weight"), f32("backbone.embed.bias")}; weights.input_norm = norm("backbone.norm");