From 0e6d58a1545f025c3da564587a7aab44cc7e5045 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 00:30:22 +0900 Subject: [PATCH 1/9] feat(model): add Confucius4-R2T2 real-time streaming ASR (r2t2_asr) Port NetEase Youdao Confucius4-R2T2 (a Qwen3-ASR-1.7B fine-tune with Longest Stable Prefix decoding) as a community model family. Runtime * src/community_models/r2t2_asr + include/engine/community_models/r2t2_asr: assets, Whisper log-mel frontend, windowed audio tower, LSP streaming state machine, and the R2T2 text pipeline (punctuation-by-context, repetition repair, language-tag parsing, "|" truncation). * Schema-v1 spec-backed loader: model_specs/r2t2_asr.json is the single source of truth for metadata, capabilities, options and packages, and the loader factory lives next to the session, so the family ships no loader.{h,cpp}. * The thinker is a thin adapter over the shared runtime::GreedyQwenDecoderRuntime (graph lifetimes, audio-embedding injection, static KV cache); long audio uses engine::audio::plan_audio_chunks and the tower reuses the shared modules plus core::ensure_backend_addressable_layout. No framework files are modified. * Offline and append-only streaming modes. Committed deltas follow the upstream WebSocket integrator contract (slice by code-point length, authoritative final transcript); chunk sizes 80-2000 ms via r2t2_asr.chunk_size_ms. * Language accepts ISO-639 codes (zh/en/ja/...) or canonical prompt names (Chinese/English/...); Auto detects. Checkpoints below Q8_0 are rejected at load time with an actionable error. Packaging and UI * model_specs/r2t2_asr.json (schema v1, status community) with the HF safetensors package plus published Q8_0 and F16 GGUF packages (davidxifeng/Confucius4-R2T2-gguf). Each GGUF embeds its sidecars and the schema-v1 option contract, so one file is a fully standalone package. * webui: catalog entry, r2t2_asr session controls, GGUF install choices, live microphone transcription, and the language dropdown. * docs/community_models/r2t2.md, the ASR model table, app/server/example.json. Verification (macOS MPS reference from the upstream repository) * tests/r2t2_asr: MPS golden generator, per-chunk comparison harness, and a repo-native offline+streaming smoke test. * Chinese reference clip: offline text, committed delta stream and final transcript are exact, 21/21 committed prefixes identical (auto and forced language). * English clip: offline, committed stream and final transcript exact; two internal prefixes differ by one token, which the reference itself reproduces between its own fp16 and bf16 runs. * Q8_0 and F16 GGUF reproduce the same transcripts; 48 kHz stereo input goes through the shared mono conversion/resampling helper with identical results; server offline, SSE streaming and live-PCM paths verified; model unload returns RSS from 6.77 GiB to 0.20 GiB. --- CMakeLists.txt | 30 + app/server/example.json | 18 + docs/asr.md | 23 + docs/community_models/r2t2.md | 341 ++++++++++ .../engine/community_models/r2t2_asr/assets.h | 91 +++ .../community_models/r2t2_asr/audio_encoder.h | 35 + .../r2t2_asr/frontend_whisper.h | 22 + .../community_models/r2t2_asr/prompt_asr.h | 18 + .../community_models/r2t2_asr/session.h | 123 ++++ .../r2t2_asr/text_postprocess.h | 77 +++ .../community_models/r2t2_asr/thinker.h | 40 ++ .../r2t2_asr/tokenizer_text.h | 43 ++ .../engine/community_models/r2t2_asr/types.h | 74 +++ model_specs/r2t2_asr.json | 296 +++++++++ src/community_models/r2t2_asr/assets.cpp | 280 ++++++++ .../r2t2_asr/audio_encoder.cpp | 615 ++++++++++++++++++ .../r2t2_asr/frontend_whisper.cpp | 93 +++ src/community_models/r2t2_asr/prompt_asr.cpp | 12 + src/community_models/r2t2_asr/session.cpp | 602 +++++++++++++++++ .../r2t2_asr/text_postprocess.cpp | 605 +++++++++++++++++ src/community_models/r2t2_asr/thinker.cpp | 119 ++++ .../r2t2_asr/tokenizer_text.cpp | 115 ++++ tests/r2t2_asr/README.md | 63 ++ tests/r2t2_asr/compare.py | 178 +++++ tests/r2t2_asr/golden.json | 77 +++ tests/r2t2_asr/golden_sample16k.json | 229 +++++++ tests/r2t2_asr/golden_sample16k_bf16.json | 230 +++++++ tests/r2t2_asr/golden_zh.json | 77 +++ tests/r2t2_asr/make_golden.py | 103 +++ .../r2t2_asr/test_r2t2_asr_transcription.cpp | 212 ++++++ webui/configs/model_params.json | 8 + webui/configs/models_catalog.json | 2 + webui/native/dist/index.html | 78 +-- webui/native/src/lib/catalog.ts | 1 + webui/native/src/routes/+page.svelte | 10 +- 35 files changed, 4898 insertions(+), 42 deletions(-) create mode 100644 docs/community_models/r2t2.md create mode 100644 include/engine/community_models/r2t2_asr/assets.h create mode 100644 include/engine/community_models/r2t2_asr/audio_encoder.h create mode 100644 include/engine/community_models/r2t2_asr/frontend_whisper.h create mode 100644 include/engine/community_models/r2t2_asr/prompt_asr.h create mode 100644 include/engine/community_models/r2t2_asr/session.h create mode 100644 include/engine/community_models/r2t2_asr/text_postprocess.h create mode 100644 include/engine/community_models/r2t2_asr/thinker.h create mode 100644 include/engine/community_models/r2t2_asr/tokenizer_text.h create mode 100644 include/engine/community_models/r2t2_asr/types.h create mode 100644 model_specs/r2t2_asr.json create mode 100644 src/community_models/r2t2_asr/assets.cpp create mode 100644 src/community_models/r2t2_asr/audio_encoder.cpp create mode 100644 src/community_models/r2t2_asr/frontend_whisper.cpp create mode 100644 src/community_models/r2t2_asr/prompt_asr.cpp create mode 100644 src/community_models/r2t2_asr/session.cpp create mode 100644 src/community_models/r2t2_asr/text_postprocess.cpp create mode 100644 src/community_models/r2t2_asr/thinker.cpp create mode 100644 src/community_models/r2t2_asr/tokenizer_text.cpp create mode 100644 tests/r2t2_asr/README.md create mode 100644 tests/r2t2_asr/compare.py create mode 100644 tests/r2t2_asr/golden.json create mode 100644 tests/r2t2_asr/golden_sample16k.json create mode 100644 tests/r2t2_asr/golden_sample16k_bf16.json create mode 100644 tests/r2t2_asr/golden_zh.json create mode 100644 tests/r2t2_asr/make_golden.py create mode 100644 tests/r2t2_asr/test_r2t2_asr_transcription.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c325d1498..708f40034 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1469,6 +1469,22 @@ audiocpp_add_model(qwen3_asr qwen3_forced_aligner ) +audiocpp_add_model(r2t2_asr + SOURCES + src/community_models/r2t2_asr/assets.cpp + src/community_models/r2t2_asr/text_postprocess.cpp + src/community_models/r2t2_asr/tokenizer_text.cpp + src/community_models/r2t2_asr/frontend_whisper.cpp + src/community_models/r2t2_asr/audio_encoder.cpp + src/community_models/r2t2_asr/thinker.cpp + src/community_models/r2t2_asr/prompt_asr.cpp + src/community_models/r2t2_asr/session.cpp + INCLUDES + engine/community_models/r2t2_asr/session.h + LOADERS + engine::community_models::r2t2_asr::make_r2t2_asr_loader +) + audiocpp_add_model(qwen3_forced_aligner SOURCES src/models/qwen3_forced_aligner/processor.cpp @@ -2971,6 +2987,20 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST target_link_libraries(test_granite5asr_golden_transcription PRIVATE OpenMP::OpenMP_CXX) endif() + if (r2t2_asr IN_LIST AUDIOCPP_LINKED_MODELS) + add_executable(test_r2t2_asr_transcription + tests/r2t2_asr/test_r2t2_asr_transcription.cpp + ) + target_compile_definitions(test_r2t2_asr_transcription PRIVATE + ENGINE_REPO_ROOT="${CMAKE_CURRENT_SOURCE_DIR}" + ) + target_link_libraries(test_r2t2_asr_transcription PRIVATE engine_runtime ggml) + target_include_directories(test_r2t2_asr_transcription PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + if (ENGINE_ENABLE_OPENMP) + target_link_libraries(test_r2t2_asr_transcription PRIVATE OpenMP::OpenMP_CXX) + endif() + endif() + if (vibeasr IN_LIST AUDIOCPP_LINKED_MODELS) add_executable(test_vibeasr_vae_encoder tests/vibeasr/test_vibeasr_vae_encoder.cpp diff --git a/app/server/example.json b/app/server/example.json index 116901419..9ac63be64 100644 --- a/app/server/example.json +++ b/app/server/example.json @@ -36,6 +36,24 @@ "path": "../../models/Qwen3-ASR-0.6B", "task": "asr", "mode": "offline" + }, + { + "id": "r2t2-asr", + "family": "r2t2_asr", + "path": "../../models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", + "task": "asr", + "mode": "offline" + }, + { + "id": "r2t2-asr-stream", + "family": "r2t2_asr", + "path": "../../models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", + "task": "asr", + "mode": "streaming", + "session_options": { + "r2t2_asr.chunk_size_ms": "320", + "r2t2_asr.max_tokens": "32" + } } ] } diff --git a/docs/asr.md b/docs/asr.md index c2957955d..5b286567a 100644 --- a/docs/asr.md +++ b/docs/asr.md @@ -7,6 +7,7 @@ | Fun-ASR-Nano | `fun_asr_nano` | offline | [Fun-ASR-Nano](#fun-asr-nano) | | Granite Speech 5.0 TurboCTC | `granite5asr` | offline | [Granite Speech 5.0 TurboCTC](community_models/granite5asr.md) | | Qwen3 ASR | `qwen3_asr` | offline, streaming | [Qwen3 ASR](#qwen3-asr) | +| Confucius4-R2T2 | `r2t2_asr` | offline, streaming | [Confucius4-R2T2](community_models/r2t2.md) | | Citrinet ASR | `citrinet_asr` | offline | [Citrinet ASR](#citrinet-asr) | | Kroko Community ASR | `kroko_asr` | offline, streaming | [Kroko Community ASR](#kroko-community-asr) | | Higgs Audio STT | `higgs_audio_stt` | offline, streaming | [Higgs Audio STT](models/higgs_audio_stt.md) | @@ -61,6 +62,28 @@ audiocpp_cli --task asr --family qwen3_asr --model models/Qwen3-ASR-1.7B-hf --ba audiocpp_cli --task asr --mode streaming --family qwen3_asr --model models/Qwen3-ASR-1.7B-hf --backend cuda --audio speech_16k.wav --request-option audio_chunk_seconds=5 --text-out transcript.txt ``` +## Confucius4-R2T2 + +Confucius4-R2T2 is a low-latency append-only streaming ASR model: a Qwen3-ASR +1.7B fine-tune with Longest Stable Prefix (LSP) decoding. Committed text is +never revised, and chunk sizes from 80 ms to 2 s are supported. It runs the +same audio tower as Qwen3 ASR, so only the streaming state machine differs. + +```bash +audiocpp_cli --task asr --family r2t2_asr --model models/Confucius4-R2T2 \ + --backend metal --audio speech_16k.wav --text-out transcript.txt +``` + +```bash +audiocpp_cli --task asr --mode streaming --family r2t2_asr \ + --model models/Confucius4-R2T2 --backend metal --audio speech_16k.wav \ + --session-option r2t2_asr.chunk_size_ms=320 --text-out transcript.txt +``` + +Streaming emits append-only partial text; the final transcript is returned when +the stream ends. See the [Confucius4-R2T2 model guide](community_models/r2t2.md) for the +session options, the LSP state machine, and the MPS golden verification recipe. + ## Citrinet ASR Citrinet is an offline CTC ASR model. It produces transcription text from speech audio. diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md new file mode 100644 index 000000000..6a5712339 --- /dev/null +++ b/docs/community_models/r2t2.md @@ -0,0 +1,341 @@ +# Confucius4-R2T2 + +Confucius4-R2T2 (Real Real-Time Transcription) is NetEase Youdao's low-latency, +append-only streaming ASR model. It is a Qwen3-ASR-1.7B fine-tune that keeps the +same audio tower and thinker graph, and adds a Longest Stable Prefix (LSP) +decoding algorithm: every chunk re-decodes the whole accumulated audio with the +previously recognized text as a continuation prompt, and only the stable prefix +of the new result is committed downstream. Committed text is never revised, +which is what makes it suitable for live captioning and downstream agents. + +| Field | Value | +|---|---| +| Family | `r2t2_asr` | +| HF checkpoint | `netease-youdao/Confucius4-R2T2` | +| Task | `asr` | +| Modes | `offline`, `streaming` | +| Input | 16 kHz speech WAV (other rates are resampled by the frontend) | +| Output | Transcript text | +| Streaming output | Append-only partial text plus the final transcript | +| Timestamps | Not supported | +| Context / hotwords | Optional `--text` system prompt | + +## Independent implementation + +`r2t2_asr` is a **standalone family** (a community port, `status: community` in +the spec): assets, Whisper log-mel frontend, windowed audio tower, tokenizer, LSP +session, and text post-processing live under +`src/community_models/r2t2_asr/` + `include/engine/community_models/r2t2_asr/` and +share no code with the `qwen3_asr` family, so the two can evolve independently. +The loader itself is the framework's **schema-v1 spec-backed loader** — +`model_specs/r2t2_asr.json` is the single source of truth for metadata, +capabilities, options and packages, and the factory lives next to the session +(`make_r2t2_asr_loader`), so there is no per-model `loader.{h,cpp}`. What it does +**not** duplicate is framework infrastructure: + +* the thinker is a thin adapter over the shared greedy Qwen decoder runtime + (`runtime::GreedyQwenDecoderRuntime`), which owns the prefill/decode graphs, + the audio-embedding injection, the static KV cache, and greedy sampling; +* the audio tower builds on the shared modules (`Conv2dModule`, `GeluModule`, + `LinearModule`, `LayerNormModule`, `ScaledDotProductAttentionModule`) and the + shared layout/contiguity helper (`core::ensure_backend_addressable_layout`); +* long-audio chunking uses the shared audio chunk planner + (`engine::audio::plan_audio_chunks` + `slice_audio_buffer`); +* multichannel and off-rate input goes through the shared mono conversion and + resampling helper inside the frontend. + +The family-specific pieces are: + +* `session.cpp` — offline transcription plus the LSP streaming state machine, +* `text_postprocess.cpp` — `normalize_punct_by_context`, repetition repair, + `language X` parsing, Chinese spacing, and `|` truncation, +* `assets.cpp` — checkpoint resolution that keeps symlinked weight files + loadable and rejects precisions below Q8_0 (see below), +* `thinker.cpp` — maps the R2T2 config and `thinker.*` tensor layout onto the + shared decoder, including the tied LM head the checkpoint ships without. + +### Rope note + +The R2T2 config declares interleaved mrope with `mrope_section [24, 20, 20]`, but +the model only ever sees audio and text, so all three position streams are +identical and mrope degenerates to standard NEOX RoPE. The shared decoder's +NEOX rope therefore reproduces the reference numerics, which the golden checks +in `tests/r2t2_asr/` confirm chunk by chunk. + +## Install + +```bash +python3 tools/model_manager_v2.py install r2t2_asr_safetensors +``` + +Or point `--model` at any directory with the HF checkpoint layout (`config.json`, +`generation_config.json`, `preprocessor_config.json`, `model.safetensors`, +tokenizer files). Hugging Face cache snapshot directories work directly, even +though their `model.safetensors` is a symlink into the blob store: the family +resolves the checkpoint itself instead of going through the shared canonicalized +tensor path. + +## Offline transcription + +```bash +audiocpp_cli --task asr --family r2t2_asr \ + --model models/Confucius4-R2T2 --backend metal \ + --audio speech_16k.wav --text-out transcript.txt +``` + +Audio longer than 30 seconds is split into 30-second chunks inside the session +and the chunk transcripts are joined with a single space. Pass `--language +Chinese` (or any supported language) to skip language detection, and `--text +"hotword, another term"` to bias recognition with a context prompt. + +## Streaming transcription + +```bash +audiocpp_cli --task asr --mode streaming --family r2t2_asr \ + --model models/Confucius4-R2T2 --backend metal \ + --audio speech_16k.wav \ + --session-option r2t2_asr.chunk_size_ms=320 \ + --text-out transcript.txt +``` + +In streaming mode committed text appears as it stabilizes and the complete +transcript is returned when the stream ends. `--audio -` streams raw 16 kHz mono +PCM from stdin for live sources. + +Chunk sizes from 80 ms to 2000 ms are supported. 320 ms is a good default on +Apple Silicon: 160 ms trades accuracy for latency and runs below real time with +eager Metal execution on smaller machines, while 2000 ms approaches offline +quality. + +### Session options (use with `--session-option`) + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `r2t2_asr.chunk_size_ms` | 80-2000 | `320` | Streaming decode chunk in milliseconds. | +| `r2t2_asr.unfixed_chunk_num` | integer | `2` | Leading chunks decoded without a stable-prefix prompt. | +| `r2t2_asr.unfixed_token_num` | integer | `5` | Tokens rolled back from the accumulated text before it is used as the prefix prompt. | +| `r2t2_asr.rollback_punctuation` | `true`, `false` | `false` | Keep trailing text uncommitted when it already ends with punctuation instead of rolling back tokens. | +| `r2t2_asr.max_tokens` | integer | `32` | Greedy decode budget per chunk and for the final flush. | +| `r2t2_asr.audio_encoder_weight_type` | `native`, `f32`, `f16` | `native` | Audio tower weight storage. | +| `r2t2_asr.thinker_weight_type` (alias `r2t2_asr.weight_type`) | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Thinker weight storage. | +| `r2t2_asr.audio_encoder_graph_arena_mb` | MB | `128` | Audio tower graph arena. | +| `r2t2_asr.thinker_prefill_graph_arena_mb` | MB | `256` | Thinker prefill graph arena. | +| `r2t2_asr.thinker_decode_graph_arena_mb` | MB | `256` | Thinker decode graph arena. | +| `r2t2_asr.thinker_weight_context_mb` | MB | `64` | Thinker weight context. | + +### Request options (use with `--request-option`) + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `language` (or `--language`) | language name | empty | Force the recognition language; `Auto` keeps detection. | +| `max_tokens` (or `--max-tokens`) | integer | model config | Offline decode budget. | + +## How the streaming state machine works + +Each full chunk: + +1. Appends the chunk to the accumulated audio (nothing is dropped, no padding). +2. Builds the prompt as `chat template + stable prefix`, where the stable prefix + is the accumulated transcript minus `unfixed_token_num` tokens. Before + `unfixed_chunk_num` chunks the prefix is empty. Decoding a prefix never cuts + a multi-byte token: the rollback grows until the decoded prefix contains no + U+FFFD replacement character. +3. Greedy-decodes with a `max_new_tokens` budget on the accumulated audio. +4. Normalizes punctuation by context, reparses the `language Xtext` + output, and re-derives the stable prefix (again minus + `unfixed_token_num` tokens). +5. Reports the growth of the stable prefix as committed text. + +`finish_stream` flushes any tail shorter than one chunk with a fixed rollback, +then returns the complete transcript. + +### Committed-text contract + +Committed deltas follow the upstream WebSocket integrator exactly: + +```python +if len(fixed_text) > len(last_fixed_text): + emit fixed_text[len(last_fixed_text):] + last_fixed_text = fixed_text +``` + +Two consequences are inherited from the reference implementation and are +deliberate: + +* Lengths are counted in code points, so a delta is always valid UTF-8. +* The stable prefix can regress between chunks — a token rollback can leave a + partial metadata fragment such as `language` at the head of the committed + text, after which the next deltas slice past it. The authoritative transcript + is always delivered again in the final result (`transcript.text.done` on the + server), so consumers that need exact text should use that. + +The reference also ships a rolling-window variant for unbounded streams ("no +reset": keep 16 s of audio, discard the oldest 8 s and the matching text). This +port implements the standard variant used by the upstream WebSocket server, +which bounds audio per utterance with VAD. Because the audio tower uses 1500 +positions, a single unsegmented stream is limited to roughly 110 s of +accumulated audio; segment longer streams (as the reference server does) or add +the rolling-window variant. + +## Server usage + +`server.local.json` declares `r2t2-asr` (offline) and `r2t2-asr-stream` +(streaming, 320 ms chunks): + +```bash +curl http://127.0.0.1:8488/v1/audio/transcriptions \ + -F model=r2t2-asr -F file=@speech.wav +``` + +```bash +# Decoding deltas of an uploaded file +curl -N http://127.0.0.1:8488/v1/audio/transcriptions \ + -F model=r2t2-asr-stream -F stream=true -F file=@speech.wav +``` + +```bash +# Live PCM: deltas appear while the audio is still arriving +ffmpeg -f avfoundation -i ":0" -ar 16000 -ac 1 -f s16le - \ + | curl -N -X POST -H 'Expect:' -T - \ + 'http://127.0.0.1:8488/v1/audio/transcriptions/live?model=r2t2-asr-stream&sample_rate=16000&channels=1&sample_format=s16le' +``` + +## GGUF checkpoints + +GGUF is supported for Q8_0 and higher precision. `f16`, `q8_0`, and the native +bf16 safetensors checkpoint all reproduce the reference transcripts exactly; 4- +and 5-bit quantizations (legacy and k-quant) are rejected at load time with an +actionable error instead of silently decoding to empty text, because this +graph's kernels are not validated below Q8_0. + +Verified conversions are published at +[`davidxifeng/Confucius4-R2T2-gguf`](https://huggingface.co/davidxifeng/Confucius4-R2T2-gguf) +(a quantized derivative work, distributed under the upstream NetEase Youdao +model license — see the repository's `LICENSE`, `LICENSE_zh`, and `NOTICE`): + +| Package id | File | Quantization | Size | +|---|---|---|---:| +| `r2t2_asr_q8_0` (default) | `r2t2-q8_0.gguf` | Q8_0 | 2.31 GiB | +| `r2t2_asr_f16` | `r2t2-f16.gguf` | F16 | 3.81 GiB | + +```bash +python3 tools/model_manager_v2.py install r2t2_asr_q8_0 # or r2t2_asr_f16 +``` + +These files are self-contained: tokenizer, processor/generation config, chat +template and the model spec are embedded, and the embedded spec carries the +**schema-v1 option contract**, so option validation comes from the file itself +rather than from a spec installed beside the runtime (a legacy embedded spec +triggers a `[warning][model_spec]` fallback instead). + +### Converting a checkpoint yourself + +Convert a real (non-symlinked) checkpoint directory: + +```bash +audiocpp_gguf \ + --input /path/to/Confucius4-R2T2/model.safetensors \ + --root /path/to/Confucius4-R2T2 \ + --family r2t2_asr \ + --output models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf \ + --type q8_0 +``` + +The converter resolves sidecars from a real directory, so run it against a +directory whose files are not symlinks (a plain `cp -r` of the checkpoint, or a +directory created by the model manager). Direct loading of a symlinked HF cache +snapshot is supported and does not need this step. The output embeds the +sidecars and the model spec, so the single `.gguf` is portable: + +```bash +audiocpp_cli --task asr --family r2t2_asr \ + --model models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf --backend metal \ + --audio speech_16k.wav +``` + +Loading accepts either the `.gguf` file, a directory holding one, or a directory +holding both formats — a GGUF checkpoint wins when both are present, matching +the Qwen-family convention. + +The published checkpoint ties the LM head to the token embedding and therefore +contains no `lm_head.weight`; the family detects that and reuses the embedding, +so conversions need no special flags. + +### Verified GGUF results + +| Checkpoint | Size | Offline | Committed stream | Final transcript | Per-chunk | +|---|---:|---|---|---|---| +| `model.safetensors` (bf16) | 3.80 GiB | exact | exact | exact | 21/21 | +| `--type f16` GGUF | 3.81 GiB | exact | exact | exact | 21/21 | +| `--type q8_0` GGUF | 2.31 GiB | exact | exact | exact | 21/21 | +| `--type q4_k` / `q4_0` | — | rejected at load with a clear error | | | | + +No degradation is measurable at Q8_0 on the verification clip; it is the +recommended distribution format and the default package for this family. + +## Verification + +The port is verified against the macOS MPS reference with golden traces: + +```bash +# 1. Golden from the reference implementation (in the Confucius4-R2T2 repo) +cd /path/to/Confucius4-R2T2 +PYTHONPATH=. uv run python /path/to/audio.cpp/tests/r2t2_asr/make_golden.py \ + --model_path /path/to/audio.cpp/models/Confucius4-R2T2 \ + --audio resources/test.wav \ + --out /path/to/audio.cpp/tests/r2t2_asr/golden.json + +# 2. Compare the C++ runtime (offline text, per-chunk committed text, final text) +cd /path/to/audio.cpp +python3 tests/r2t2_asr/compare.py \ + --cli build/macos-metal-release/bin/audiocpp_cli \ + --model models/Confucius4-R2T2 \ + --audio /tmp/test.wav \ + --golden tests/r2t2_asr/golden.json --backend metal +``` + +`compare.py` runs the CLI with trace logging, parses the per-chunk trace, and +diffs it against the golden. A repo-native smoke test runs both paths without +the Python environment: + +```bash +build/macos-metal-release/bin/test_r2t2_asr_transcription --backend metal +``` + +(It skips with exit code 125 when `models/Confucius4-R2T2` or the audio asset is +absent.) + +### Results on the reference machine + +`compare.py` checks four things per golden: the offline transcript, the +**committed delta stream**, the final streaming transcript, and every per-chunk +`fixed_text`. + +| Golden | Offline text | Committed stream | Final transcript | Per-chunk `fixed_text` | +|---|---|---|---|---| +| `golden.json` (upstream Chinese `resources/test.wav`) | exact | exact | exact | 21/21 exact | +| `golden_zh.json` (same audio, `--language Chinese`) | exact | exact | exact | 21/21 exact | +| `golden_sample16k.json` (`assets/resources/sample_16k.wav`, English) | exact | exact | exact | 41/43 exact | +| `golden_sample16k_bf16.json` (same audio, bf16 reference) | exact | exact | exact | 41/43 exact | + +Everything a client observes is identical to the reference on every clip: the +offline transcript, the committed delta stream (including the reference's +`language` metadata-fragment artifact), and the final transcript. + +### The two English chunks that differ internally + +On the 14 s English clip, chunks 37 and 42 differ in `fixed_text` by about one +token of the trailing numeric span (`22,00` vs `22,0`, and a trailing +` you.` predicted one chunk earlier). This is greedy-decoding sensitivity, not +port logic: + +* both differences appear in the raw decoded text, before any R2T2 + post-processing or rollback runs; +* the reference disagrees with **itself** at the same order of magnitude — its + fp16 and bf16 runs differ at chunks 13 and 37; +* the difference never reaches the wire, because `finish_stream` publishes no + delta and the final transcript is identical. + +So per-chunk `fixed_text` equality is exact for the Chinese reference clip and +stable to within one token for long English audio, while every observable +output is exact. diff --git a/include/engine/community_models/r2t2_asr/assets.h b/include/engine/community_models/r2t2_asr/assets.h new file mode 100644 index 000000000..f844e1188 --- /dev/null +++ b/include/engine/community_models/r2t2_asr/assets.h @@ -0,0 +1,91 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::assets { +class TensorSource; +} + +namespace engine::community_models::r2t2_asr { + +struct R2T2ASRAudioEncoderConfig { + int64_t num_mel_bins = 128; + int64_t encoder_layers = 0; + int64_t encoder_attention_heads = 0; + int64_t encoder_ffn_dim = 0; + int64_t d_model = 0; + int64_t max_source_positions = 0; + int64_t n_window = 100; + int64_t n_window_infer = 400; + int64_t conv_chunksize = 500; + int64_t downsample_hidden_size = 0; + int64_t output_dim = 0; + std::string activation_function = "gelu"; +}; + +struct R2T2ASRTextDecoderConfig { + int64_t vocab_size = 0; + int64_t output_size = 0; + int64_t hidden_size = 0; + int64_t intermediate_size = 0; + int64_t num_hidden_layers = 0; + int64_t num_attention_heads = 0; + int64_t num_key_value_heads = 0; + int64_t head_dim = 0; + int64_t max_position_embeddings = 0; + int64_t audio_token_id = 0; + int64_t audio_start_token_id = 0; + int64_t audio_end_token_id = 0; + int64_t pad_token_id = 0; + std::vector eos_token_ids; + float rms_norm_eps = 1.0e-6F; + float rope_theta = 5000000.0F; + bool attention_bias = false; + std::vector mrope_section = {24, 20, 20}; +}; + +struct R2T2ASRFrontendConfig { + int sample_rate = 16000; + int64_t feature_size = 128; + int64_t hop_length = 160; + int64_t n_fft = 400; +}; + +struct R2T2ASRConfig { + std::string model_type; + std::string thinker_model_type; + std::string model_size; + int sample_rate = 16000; + int64_t max_new_tokens = 512; + int64_t classify_num = 0; + int64_t timestamp_token_id = 0; + int64_t timestamp_segment_time_ms = 0; + bool hf_transformers_layout = false; + bool tie_word_embeddings = false; + R2T2ASRFrontendConfig frontend; + R2T2ASRAudioEncoderConfig audio_encoder; + R2T2ASRTextDecoderConfig text_decoder; + std::vector supported_languages; +}; + +struct R2T2ASRAssets { + assets::ResourceBundle resources; + R2T2ASRConfig config; + std::shared_ptr model_weights; +}; + +std::shared_ptr load_r2t2_asr_assets(const std::filesystem::path & model_path); +std::shared_ptr load_r2t2_asr_assets( + const std::filesystem::path & model_path, + std::string_view package_family); +std::shared_ptr load_r2t2_asr_assets( + assets::ResourceBundle resources); + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/audio_encoder.h b/include/engine/community_models/r2t2_asr/audio_encoder.h new file mode 100644 index 000000000..7e70677ef --- /dev/null +++ b/include/engine/community_models/r2t2_asr/audio_encoder.h @@ -0,0 +1,35 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/r2t2_asr/assets.h" +#include "engine/community_models/r2t2_asr/types.h" + +#include +#include + +namespace engine::community_models::r2t2_asr { + +class R2T2ASRAudioEncoderGraph; +struct R2T2ASRAudioEncoderWeights; + +class R2T2ASRAudioEncoderRuntime { +public: + R2T2ASRAudioEncoderRuntime( + std::shared_ptr assets, + core::ExecutionContext & execution, + size_t graph_arena_bytes, + assets::TensorStorageType weight_storage_type); + ~R2T2ASRAudioEncoderRuntime(); + + R2T2ASRAudioEmbeddings encode(const R2T2ASRAudioFeatures & features); + +private: + std::shared_ptr assets_; + std::shared_ptr weights_; + core::ExecutionContext * execution_ = nullptr; + size_t graph_arena_bytes_ = 0; + std::unique_ptr graph_; +}; + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/frontend_whisper.h b/include/engine/community_models/r2t2_asr/frontend_whisper.h new file mode 100644 index 000000000..713183301 --- /dev/null +++ b/include/engine/community_models/r2t2_asr/frontend_whisper.h @@ -0,0 +1,22 @@ +#pragma once + +#include "engine/framework/audio/dsp.h" +#include "engine/community_models/r2t2_asr/assets.h" +#include "engine/community_models/r2t2_asr/types.h" + +#include + +namespace engine::community_models::r2t2_asr { + +class R2T2ASRWhisperFrontend { +public: + explicit R2T2ASRWhisperFrontend(std::shared_ptr assets); + + R2T2ASRAudioFeatures extract(const runtime::AudioBuffer & audio) const; + +private: + std::shared_ptr assets_; + engine::audio::WhisperLogMelExtractor extractor_; +}; + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/prompt_asr.h b/include/engine/community_models/r2t2_asr/prompt_asr.h new file mode 100644 index 000000000..cbc5a8ddc --- /dev/null +++ b/include/engine/community_models/r2t2_asr/prompt_asr.h @@ -0,0 +1,18 @@ +#pragma once + +#include "engine/community_models/r2t2_asr/tokenizer_text.h" +#include "engine/community_models/r2t2_asr/types.h" + +namespace engine::community_models::r2t2_asr { + +class R2T2ASRPromptBuilder { +public: + explicit R2T2ASRPromptBuilder(const R2T2ASRTextTokenizer & tokenizer); + + R2T2ASRPrompt build(const R2T2ASRRequest & request, int64_t audio_feature_tokens) const; + +private: + const R2T2ASRTextTokenizer & tokenizer_; +}; + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/session.h b/include/engine/community_models/r2t2_asr/session.h new file mode 100644 index 000000000..dc176c791 --- /dev/null +++ b/include/engine/community_models/r2t2_asr/session.h @@ -0,0 +1,123 @@ +#pragma once + +#include "engine/framework/runtime/session_base.h" +#include "engine/framework/runtime/model.h" +#include "engine/community_models/r2t2_asr/assets.h" +#include "engine/community_models/r2t2_asr/audio_encoder.h" +#include "engine/community_models/r2t2_asr/frontend_whisper.h" +#include "engine/community_models/r2t2_asr/thinker.h" +#include "engine/community_models/r2t2_asr/tokenizer_text.h" +#include "engine/community_models/r2t2_asr/types.h" + +#include +#include +#include +#include +#include + +namespace engine::community_models::r2t2_asr { + +/// Spec-backed loader factory (schema-v1 contract): the framework derives +/// metadata, capabilities and option validation from model_specs/r2t2_asr.json, +/// so this family ships no per-model loader.{h,cpp}. +std::shared_ptr make_r2t2_asr_loader(); + +/// Streaming decode configuration; defaults mirror +/// R2T2ASRModel.init_streaming_state() in the reference implementation. +struct R2T2ASRStreamConfig { + double chunk_seconds = 0.32; + int64_t unfixed_chunk_num = 2; + int64_t unfixed_token_num = 5; + bool rollback_punctuation = false; + int64_t max_new_tokens = 32; +}; + +/// Confucius4-R2T2 streaming ASR session. +/// +/// This family owns its full Qwen3-ASR-derived graph (audio tower, thinker, +/// tokenizer) plus two execution paths that the plain Qwen3-ASR family does not +/// have: +/// +/// * offline transcription with R2T2 text parsing, and +/// * Longest Stable Prefix (LSP) streaming: every chunk re-decodes the whole +/// accumulated audio with a prompt carrying the previously recognized text +/// minus a small token rollback, and only the stable prefix of the result is +/// committed downstream (append-only). +class R2T2ASRSession final + : public runtime::RuntimeSessionBase + , public runtime::IOfflineVoiceTaskSession + , public runtime::IStreamingVoiceTaskSession { +public: + R2T2ASRSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets); + ~R2T2ASRSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + + runtime::TaskResult run(const runtime::TaskRequest & request) override; + + runtime::StreamingPolicy streaming_policy() const override; + void start_stream(const runtime::TaskRequest & request) override; + void set_stream_event_sink(runtime::StreamEventCallback sink) override; + void reset() override; + runtime::StreamEvent process_audio_chunk(const runtime::AudioChunk & chunk) override; + runtime::TaskResult finish_stream() override; + runtime::TaskResult finalize() override; + +private: + struct StreamOutcome { + std::string text; + std::string fixed_text; + }; + + R2T2ASRRequest make_request(const runtime::TaskRequest & request) const; + R2T2ASRResult run_single(const R2T2ASRRequest & request); + std::string generate_text(const R2T2ASRPrompt & prompt, const R2T2ASRAudioEmbeddings & embeddings); + + StreamOutcome decode_stream_chunk(bool final_flush); + std::string build_stream_prefix(bool final_flush) const; + std::string decode_rollback_prefix(const std::vector & ids, int64_t rollback) const; + void publish_stream_delta(const std::string & fixed_text, runtime::StreamEvent & event); + + runtime::TaskSpec task_; + std::shared_ptr assets_; + R2T2ASRStreamConfig stream_config_; + size_t audio_encoder_graph_arena_bytes_ = 128ull * 1024ull * 1024ull; + size_t thinker_prefill_graph_arena_bytes_ = 256ull * 1024ull * 1024ull; + size_t thinker_decode_graph_arena_bytes_ = 256ull * 1024ull * 1024ull; + size_t thinker_weight_context_bytes_ = 64ull * 1024ull * 1024ull; + engine::assets::TensorStorageType audio_encoder_weight_storage_type_ = engine::assets::TensorStorageType::Native; + engine::assets::TensorStorageType thinker_weight_storage_type_ = engine::assets::TensorStorageType::Native; + + R2T2ASRTextTokenizer tokenizer_; + R2T2ASRWhisperFrontend frontend_; + R2T2ASRAudioEncoderRuntime audio_encoder_; + R2T2ASRThinkerRuntime thinker_; + + // Streaming state (mirrors ASRStreamingState in the reference code). + runtime::TaskRequest streaming_request_; + runtime::TaskResult streaming_result_; + std::string prompt_raw_; + std::string force_language_; + std::string context_; + std::string language_; + std::string text_; + std::string raw_decoded_; + std::vector buffer_; + std::vector audio_accum_; + int64_t chunk_size_samples_ = 0; + int64_t chunk_id_ = 0; + size_t published_codepoints_ = 0; + int stream_sample_rate_ = 0; + int stream_channels_ = 1; + runtime::StreamEventCallback stream_event_sink_; + bool stream_started_ = false; + std::chrono::steady_clock::time_point stream_wall_start_{}; +}; + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/text_postprocess.h b/include/engine/community_models/r2t2_asr/text_postprocess.h new file mode 100644 index 000000000..fc2b11a5a --- /dev/null +++ b/include/engine/community_models/r2t2_asr/text_postprocess.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include + +namespace engine::community_models::r2t2_asr { + +// Faithful C++ ports of the R2T2 text post-processing pipeline +// (r2t2/r2t2_asr.py and qwen_asr/inference/utils.py). All functions operate +// on UTF-8 text and mirror the Python string semantics so the streaming +// state machine reproduces the reference outputs token for token. + +inline constexpr const char * kAsrTextTag = ""; +inline constexpr const char * kLanguagePrefix = "language "; +inline constexpr const char * kReplacementChar = "\xEF\xBF\xBD"; + +struct R2T2ParsedOutput { + std::string language; + std::string text; +}; + +/// Replaces every maximal invalid UTF-8 subsequence with U+FFFD, mirroring +/// the replacement characters HuggingFace decode produces for truncated +/// multibyte tokens. +std::string sanitize_utf8_lossy(const std::string & text); + +/// R2T2 punctuation normalization: each punctuation mark is rewritten to the +/// Chinese or ASCII variant based on the nearest preceding character. +std::string normalize_punct_by_context(const std::string & text); + +/// qwen_asr detect_and_fix_repetitions: collapse degenerate character and +/// pattern repetition hallucinations (threshold counted in code points). +std::string detect_and_fix_repetitions(const std::string & text, int threshold = 20); + +/// Removes whitespace runs between two Chinese characters. +std::string remove_spaces_between_chinese(const std::string & text); + +/// First letter uppercase, remaining letters lowercase ("cHINese" -> "Chinese"). +std::string normalize_language_name(const std::string & language); + +/// Maps the ISO-639 style codes used by the model spec (and the UI) onto the +/// canonical language names the R2T2 prompt expects, e.g. "zh" -> "Chinese", +/// "en" -> "English". Empty input and "auto"/"Auto" mean automatic language +/// detection. Unrecognised values are passed through normalize_language_name so +/// callers can still validate them against the model's supported list. +std::string resolve_language(const std::string & language); + +/// R2T2 parse_language_output: extracts the detected language from raw output +/// that is expected to carry "language Xtext". +R2T2ParsedOutput parse_language_output(const std::string & raw, const std::string & user_language); + +/// qwen_asr parse_asr_output with repetition repair. +R2T2ParsedOutput parse_asr_output(const std::string & raw, const std::string & user_language); + +/// text.split("|")[0] +std::string truncate_at_pipe(const std::string & text); + +bool contains_asr_text_tag(const std::string & text); + +/// Part before the tag (empty when the tag is absent). +std::string text_before_asr_tag(const std::string & text); + +/// Part after the tag (empty when the tag is absent). +std::string text_after_asr_tag(const std::string & text); + +bool ends_with_rollback_punctuation(const std::string & trimmed_text); + +/// Number of Unicode code points (what Python's len() counts on str). +std::size_t utf8_codepoint_count(const std::string & text); + +/// Returns the suffix starting at `start_codepoint` (never splits a code +/// point, unlike a byte-based substr). Used to reproduce the reference +/// integrator's `fixed_text[len(last_fixed_text):]` slice. +std::string utf8_slice_from_codepoint(const std::string & text, std::size_t start_codepoint); + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/thinker.h b/include/engine/community_models/r2t2_asr/thinker.h new file mode 100644 index 000000000..dd73cd896 --- /dev/null +++ b/include/engine/community_models/r2t2_asr/thinker.h @@ -0,0 +1,40 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/r2t2_asr/assets.h" +#include "engine/community_models/r2t2_asr/types.h" + +#include +#include + +namespace engine::community_models::r2t2_asr { + +/// Greedy decoding of the Confucius4-R2T2 thinker. The Qwen3-style decoder +/// stack, the audio-embedding injection, the static KV cache, and the graph +/// lifetimes all come from the shared framework runtime +/// (runtime::GreedyQwenDecoderRuntime), so this class only maps the family's +/// config and tensor layout onto it. +class R2T2ASRThinkerRuntime { +public: + struct Impl; + + R2T2ASRThinkerRuntime( + std::shared_ptr assets, + core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + assets::TensorStorageType weight_storage_type); + ~R2T2ASRThinkerRuntime(); + + R2T2ASRGeneratedTokens generate( + const R2T2ASRPrompt & prompt, + const R2T2ASRAudioEmbeddings & audio_embeddings, + const R2T2ASRGenerationOptions & options); + +private: + std::unique_ptr impl_; +}; + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/tokenizer_text.h b/include/engine/community_models/r2t2_asr/tokenizer_text.h new file mode 100644 index 000000000..32c11f467 --- /dev/null +++ b/include/engine/community_models/r2t2_asr/tokenizer_text.h @@ -0,0 +1,43 @@ +#pragma once + +#include "engine/community_models/r2t2_asr/assets.h" +#include "engine/community_models/r2t2_asr/types.h" + +#include +#include +#include + +namespace engine::community_models::r2t2_asr { + +class R2T2ASRTextTokenizer { +public: + struct Impl; + + explicit R2T2ASRTextTokenizer(std::shared_ptr assets); + + R2T2ASRPrompt build_prompt( + const std::string & context, + const std::string & language, + int64_t audio_feature_tokens) const; + + R2T2ASRPrompt build_raw_audio_prompt( + const std::string & text, + int64_t audio_feature_tokens) const; + + /// Returns the unexpanded chat-template prompt string (audio placeholder + /// still textual). Streaming sessions append the stable-prefix + /// continuation before expanding the audio tokens. + std::string build_prompt_text(const std::string & context, const std::string & language) const; + + /// Encodes plain text with the Qwen2 BPE tokenizer (added tokens such as + /// resolve to their dedicated ids). + std::vector encode(const std::string & text) const; + + std::string decode(const std::vector & token_ids) const; + +private: + std::shared_ptr assets_; + std::shared_ptr impl_; +}; + +} // namespace engine::community_models::r2t2_asr diff --git a/include/engine/community_models/r2t2_asr/types.h b/include/engine/community_models/r2t2_asr/types.h new file mode 100644 index 000000000..fe116ef46 --- /dev/null +++ b/include/engine/community_models/r2t2_asr/types.h @@ -0,0 +1,74 @@ +#pragma once + +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include + +namespace engine::community_models::r2t2_asr { + +struct R2T2ASRGenerationOptions { + int64_t max_new_tokens = 512; + bool return_timestamps = false; + bool clamp_timestamps_to_audio = false; +}; + +struct R2T2ASRRequest { + runtime::AudioBuffer audio; + std::string context; + std::string language; + R2T2ASRGenerationOptions generation; +}; + +struct R2T2ASRResult { + std::string text; + std::string language; + std::vector word_timestamps; +}; + +struct R2T2ASRPrompt { + std::vector input_ids; + std::vector audio_token_positions; + std::vector attention_mask; +}; + +struct R2T2ASRAudioFeatures { + std::vector values; + std::vector attention_mask; + int64_t mel_bins = 0; + int64_t frames = 0; + int64_t encoder_tokens = 0; +}; + +struct R2T2ASRAudioEmbeddings { + std::vector values; + int64_t tokens = 0; + int64_t hidden_size = 0; +}; + +struct R2T2ASRGeneratedTokens { + std::vector token_ids; +}; + +inline int64_t r2t2_asr_floor_div(int64_t numerator, int64_t denominator) { + int64_t quotient = numerator / denominator; + const int64_t remainder = numerator % denominator; + if (remainder != 0 && ((remainder < 0) != (denominator < 0))) { + --quotient; + } + return quotient; +} + +inline int64_t r2t2_asr_audio_encoder_token_count(int64_t input_frames) { + if (input_frames <= 0) { + throw std::runtime_error("R2T2 ASR requires positive feature frame count"); + } + const int64_t input_lengths_leave = input_frames % 100; + const int64_t feat_lengths = r2t2_asr_floor_div(input_lengths_leave - 1, 2) + 1; + return r2t2_asr_floor_div(r2t2_asr_floor_div(feat_lengths - 1, 2) + 1 - 1, 2) + 1 + + (input_frames / 100) * 13; +} + +} // namespace engine::community_models::r2t2_asr diff --git a/model_specs/r2t2_asr.json b/model_specs/r2t2_asr.json new file mode 100644 index 000000000..c5fd19adc --- /dev/null +++ b/model_specs/r2t2_asr.json @@ -0,0 +1,296 @@ +{ + "schema_version": 1, + "family": "r2t2_asr", + "display_name": "Confucius4-R2T2", + "description": "NetEase Youdao Confucius4-R2T2 real-time ASR: a Qwen3-ASR fine-tune with Longest Stable Prefix (LSP) streaming. Low-latency append-only streaming from 80 ms to 2 s chunks, context and hotword prompts, 30 languages.", + "category": "asr", + "status": "community", + "tasks": [ + "asr" + ], + "modes": [ + "offline", + "streaming" + ], + "languages": [ + "zh", + "en", + "yue", + "ar", + "de", + "fr", + "es", + "pt", + "id", + "it", + "ko", + "ru", + "th", + "vi", + "ja", + "tr", + "hi", + "ms", + "nl", + "sv", + "da", + "fi", + "pl", + "cs", + "fil", + "fa", + "el", + "hu", + "mk", + "ro" + ], + "capabilities": { + "asr": [ + "partial_results" + ] + }, + "runtime": { + "tags": [ + "gguf", + "stream" + ] + }, + "ui": { + "recommended_package": "r2t2_asr_q8_0", + "tags": [ + "ASR", + "Stream" + ], + "docs": [ + "docs/community_models/r2t2.md", + "docs/asr.md" + ] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "netease-youdao/Confucius4-R2T2", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "r2t2_asr_q8_0", + "display_name": "Confucius4-R2T2 Q8_0 GGUF", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "Confucius4-R2T2-GGUF", + "files": [ + "r2t2-q8_0.gguf" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "davidxifeng/Confucius4-R2T2-gguf" + } + }, + { + "id": "r2t2_asr_f16", + "display_name": "Confucius4-R2T2 F16 GGUF", + "format": "gguf", + "precision": "f16", + "target_directory": "Confucius4-R2T2-GGUF", + "files": [ + "r2t2-f16.gguf" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "davidxifeng/Confucius4-R2T2-gguf" + } + }, + { + "id": "r2t2_asr_safetensors", + "display_name": "Confucius4-R2T2 (HF safetensors)", + "format": "safetensors", + "precision": "native", + "target_directory": "Confucius4-R2T2", + "files": [ + "added_tokens.json", + "chat_template.json", + "config.json", + "generation_config.json", + "merges.txt", + "model.safetensors", + "preprocessor_config.json", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "netease-youdao/Confucius4-R2T2" + } + } + ], + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer_config": "model:tokenizer_config.json" + }, + "optional_files": { + "preprocessor_config": "model:preprocessor_config.json", + "processor_config": "model:processor_config.json", + "chat_template": "model:chat_template.json", + "chat_template_jinja": "model:chat_template.jinja", + "vocab": "model:vocab.json", + "merges": "model:merges.txt", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "weights": "weights:" + } + }, + { + "format": "safetensors", + "roots": { + "model": "." + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer_config": "model:tokenizer_config.json" + }, + "optional_files": { + "preprocessor_config": "model:preprocessor_config.json", + "processor_config": "model:processor_config.json", + "chat_template": "model:chat_template.json", + "chat_template_jinja": "model:chat_template.jinja", + "vocab": "model:vocab.json", + "merges": "model:merges.txt", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "weights": "model:model.safetensors" + } + } + ], + "options": { + "request": [ + { + "name": "language", + "type": "string", + "required": false, + "default": "Auto", + "description": "Recognition language: ISO-639 code (zh/en/ja/...) or canonical name (Chinese/English/...), or Auto for detection." + }, + { + "name": "max_tokens", + "type": "int", + "required": false, + "default": 512, + "description": "Offline decode budget." + } + ], + "session": [ + { + "name": "chunk_size_ms", + "type": "int", + "required": false, + "default": 320, + "description": "Streaming decode chunk size in milliseconds (80-2000)." + }, + { + "name": "unfixed_chunk_num", + "type": "int", + "required": false, + "default": 2, + "description": "Leading chunks decoded without a stable-prefix prompt." + }, + { + "name": "unfixed_token_num", + "type": "int", + "required": false, + "default": 5, + "description": "Tokens rolled back from the accumulated text before it becomes the prefix prompt." + }, + { + "name": "rollback_punctuation", + "type": "bool", + "required": false, + "default": false, + "description": "Keep trailing text uncommitted when it ends with punctuation instead of rolling back tokens." + }, + { + "name": "max_tokens", + "type": "int", + "required": false, + "default": 32, + "description": "Greedy decode budget per streaming chunk and for the final flush (session-scoped; distinct from the offline request max_tokens)." + }, + { + "name": "audio_encoder_weight_type", + "type": "enum", + "preset": "weight_type_conv", + "required": false, + "default": "native", + "description": "Audio tower weight storage." + }, + { + "name": "thinker_weight_type", + "type": "enum", + "values": [ + "native", + "f32", + "f16", + "bf16", + "q8_0" + ], + "required": false, + "default": "native", + "description": "Thinker weight storage." + }, + { + "name": "weight_type", + "type": "enum", + "preset": "weight_type_full", + "required": false, + "default": "native", + "description": "Alias for thinker_weight_type." + }, + { + "name": "audio_encoder_graph_arena_mb", + "type": "int", + "required": false, + "default": 128, + "description": "Audio tower graph arena in MB." + }, + { + "name": "thinker_prefill_graph_arena_mb", + "type": "int", + "required": false, + "default": 256, + "description": "Thinker prefill graph arena in MB." + }, + { + "name": "thinker_decode_graph_arena_mb", + "type": "int", + "required": false, + "default": 256, + "description": "Thinker decode graph arena in MB." + }, + { + "name": "thinker_weight_context_mb", + "type": "int", + "required": false, + "default": 64, + "description": "Thinker weight context in MB." + } + ], + "load": [] + }, + "dependencies": [] +} diff --git a/src/community_models/r2t2_asr/assets.cpp b/src/community_models/r2t2_asr/assets.cpp new file mode 100644 index 000000000..5f20eb693 --- /dev/null +++ b/src/community_models/r2t2_asr/assets.cpp @@ -0,0 +1,280 @@ +#include "engine/community_models/r2t2_asr/assets.h" + +#include "engine/framework/model_spec/package.h" +#include "engine/framework/io/filesystem.h" +#include "engine/framework/io/json.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::r2t2_asr { +namespace json = engine::io::json; +namespace { + +R2T2ASRAudioEncoderConfig parse_audio_encoder_config(const json::Value & value) { + R2T2ASRAudioEncoderConfig config; + config.num_mel_bins = json::require_i64(value, "num_mel_bins"); + config.encoder_layers = json::require_i64(value, "encoder_layers"); + config.encoder_attention_heads = json::require_i64(value, "encoder_attention_heads"); + config.encoder_ffn_dim = json::require_i64(value, "encoder_ffn_dim"); + config.d_model = json::require_i64(value, "d_model"); + // The Transformers checkpoint calls a small RoPE setting + // max_position_embeddings. The audio tower still uses the same 1500-frame + // sinusoidal table as the original Qwen checkpoint. + config.max_source_positions = json::optional_i64(value, "max_source_positions", 1500); + config.n_window = json::require_i64(value, "n_window"); + config.n_window_infer = json::require_i64(value, "n_window_infer"); + config.conv_chunksize = json::require_i64(value, "conv_chunksize"); + config.downsample_hidden_size = json::require_i64(value, "downsample_hidden_size"); + config.output_dim = json::require_i64(value, "output_dim"); + config.activation_function = json::require_string(value, "activation_function"); + if (config.activation_function != "gelu") { + throw std::runtime_error("R2T2 ASR currently supports gelu audio activation"); + } + return config; +} + +R2T2ASRTextDecoderConfig parse_text_decoder_config( + const json::Value & thinker_config, + const json::Value & text_config) { + R2T2ASRTextDecoderConfig config; + config.vocab_size = json::require_i64(text_config, "vocab_size"); + config.output_size = json::optional_i64(thinker_config, "classify_num", config.vocab_size); + config.hidden_size = json::require_i64(text_config, "hidden_size"); + config.intermediate_size = json::require_i64(text_config, "intermediate_size"); + config.num_hidden_layers = json::require_i64(text_config, "num_hidden_layers"); + config.num_attention_heads = json::require_i64(text_config, "num_attention_heads"); + config.num_key_value_heads = json::require_i64(text_config, "num_key_value_heads"); + config.head_dim = json::optional_i64(text_config, "head_dim", config.hidden_size / config.num_attention_heads); + config.max_position_embeddings = json::require_i64(text_config, "max_position_embeddings"); + config.audio_token_id = json::require_i64(thinker_config, "audio_token_id"); + config.audio_start_token_id = json::optional_i64(thinker_config, "audio_start_token_id", 0); + config.audio_end_token_id = json::optional_i64(thinker_config, "audio_end_token_id", 0); + config.pad_token_id = json::optional_i64(thinker_config, "pad_token_id", config.pad_token_id); + config.pad_token_id = json::optional_i64(text_config, "pad_token_id", config.pad_token_id); + config.rms_norm_eps = json::optional_f32(text_config, "rms_norm_eps", config.rms_norm_eps); + config.rope_theta = json::optional_f32(text_config, "rope_theta", config.rope_theta); + config.attention_bias = json::optional_bool(text_config, "attention_bias", config.attention_bias); + const auto * rope_parameters = text_config.find("rope_parameters"); + if (rope_parameters != nullptr && rope_parameters->is_object()) { + config.rope_theta = json::optional_f32(*rope_parameters, "rope_theta", config.rope_theta); + } + const auto * rope_scaling = text_config.find("rope_scaling"); + if (rope_scaling != nullptr && rope_scaling->is_object()) { + config.mrope_section = json::optional_i64_array_or_scalar(*rope_scaling, "mrope_section", config.mrope_section); + } + return config; +} + +int64_t require_added_token_id(const assets::ResourceBundle & resources, std::string_view content) { + const auto tokenizer = resources.parse_json("tokenizer_json"); + for (const auto & item : tokenizer.require("added_tokens").as_array()) { + const auto * token_content = item.find("content"); + const auto * token_id = item.find("id"); + if (token_content != nullptr && token_content->is_string() && + token_id != nullptr && token_id->is_number() && token_content->as_string() == content) { + return token_id->as_i64(); + } + } + throw std::runtime_error("R2T2 ASR tokenizer.json is missing token: " + std::string(content)); +} + +void add_supported_languages(R2T2ASRConfig & config, const json::Value & root) { + static constexpr const char * kLanguages[] = { + "Chinese", "English", "Cantonese", "Arabic", "German", "French", "Spanish", "Portuguese", + "Indonesian", "Italian", "Korean", "Russian", "Thai", "Vietnamese", "Japanese", "Turkish", + "Hindi", "Malay", "Dutch", "Swedish", "Danish", "Finnish", "Polish", "Czech", "Filipino", + "Persian", "Greek", "Romanian", "Hungarian", "Macedonian", + }; + config.supported_languages = {"Auto"}; + const auto * languages = root.find("support_languages"); + if (languages != nullptr && languages->is_array()) { + for (const auto & language : languages->as_array()) { + config.supported_languages.push_back(language.as_string()); + } + return; + } + for (const char * language : kLanguages) { + config.supported_languages.emplace_back(language); + } +} + +R2T2ASRConfig parse_config(const assets::ResourceBundle & resources) { + const auto root = resources.parse_json("config"); + const auto * legacy_thinker = root.find("thinker_config"); + const bool hf_layout = legacy_thinker == nullptr; + const auto & thinker_config = hf_layout ? root : *legacy_thinker; + const auto & audio_config = thinker_config.require("audio_config"); + const auto & text_config = thinker_config.require("text_config"); + + R2T2ASRConfig config; + config.hf_transformers_layout = hf_layout; + config.model_type = json::require_string(root, "model_type"); + config.thinker_model_type = json::optional_string(thinker_config, "model_type", config.model_type); + config.model_size = json::optional_string(root, "model_size", hf_layout ? "R2T2-1.7B-hf" : config.model_type); + config.classify_num = json::optional_i64(thinker_config, "classify_num", 0); + config.timestamp_token_id = json::optional_i64(root, "timestamp_token_id", 0); + config.tie_word_embeddings = json::optional_bool( + root, + "tie_word_embeddings", + json::optional_bool(text_config, "tie_word_embeddings", false)); + config.audio_encoder = parse_audio_encoder_config(audio_config); + config.text_decoder = parse_text_decoder_config(thinker_config, text_config); + if (hf_layout) { + config.text_decoder.audio_start_token_id = require_added_token_id(resources, "<|audio_start|>"); + config.text_decoder.audio_end_token_id = require_added_token_id(resources, "<|audio_end|>"); + } + + const auto generation = resources.parse_json("generation_config"); + config.max_new_tokens = json::optional_i64(generation, "max_new_tokens", config.max_new_tokens); + config.text_decoder.pad_token_id = json::optional_i64(generation, "pad_token_id", config.text_decoder.pad_token_id); + config.text_decoder.eos_token_ids = json::require_i64_array_or_scalar(generation, "eos_token_id"); + + const auto processor = resources.parse_json(resources.has_file("processor_config") ? "processor_config" : "preprocessor_config"); + const auto * feature_extractor = processor.find("feature_extractor"); + const auto & frontend = feature_extractor != nullptr && feature_extractor->is_object() ? *feature_extractor : processor; + config.frontend.sample_rate = static_cast(json::optional_i64(frontend, "sampling_rate", config.frontend.sample_rate)); + config.frontend.feature_size = json::require_i64(frontend, "feature_size"); + config.frontend.hop_length = json::require_i64(frontend, "hop_length"); + config.frontend.n_fft = json::require_i64(frontend, "n_fft"); + config.timestamp_segment_time_ms = json::optional_i64(processor, "timestamp_segment_time", 0); + if (config.timestamp_segment_time_ms == 0) { + config.timestamp_segment_time_ms = json::optional_i64(root, "timestamp_segment_time", 0); + } + config.sample_rate = config.frontend.sample_rate; + if (config.frontend.feature_size != config.audio_encoder.num_mel_bins) { + throw std::runtime_error("R2T2 ASR frontend feature size does not match audio encoder config"); + } + + add_supported_languages(config, root); + return config; +} + +assets::ResourceBundle make_resource_bundle( + const std::filesystem::path & model_path, + std::string_view package_family) { + auto resources = engine::model_spec::load_resource_bundle_for_family(model_path, package_family); + if (!resources.has_file("preprocessor_config") && !resources.has_file("processor_config")) { + throw std::runtime_error("R2T2 ASR requires preprocessor_config.json or processor_config.json"); + } + const bool has_legacy_tokenizer = resources.has_file("vocab") && resources.has_file("merges"); + if (!has_legacy_tokenizer && !resources.has_file("tokenizer_json")) { + throw std::runtime_error("R2T2 ASR requires vocab.json plus merges.txt, or tokenizer.json"); + } + return resources; +} + +/// Locates the checkpoint without going through the resource bundle's +/// canonicalized tensor path. Canonicalizing dereferences symlinked weights +/// (Hugging Face cache snapshots, `hf download --local-dir` layouts), and the +/// blob target has no file extension, which the tensor source opener uses to +/// pick a format. Keeping this resolution inside the family avoids changing +/// shared framework behavior for every other model. +/// +/// A GGUF checkpoint wins when both formats sit in the same directory, matching +/// the convention documented for the Qwen family ("loaders prefer model.gguf +/// when both formats are present"). +std::shared_ptr open_model_weights( + const assets::ResourceBundle & resources) { + const auto & root = resources.model_root(); + if (!root.empty() && engine::io::is_existing_directory(root)) { + std::filesystem::path safetensors; + std::filesystem::path gguf; + std::error_code ec; + for (std::filesystem::recursive_directory_iterator it(root, ec), end; !ec && it != end; it.increment(ec)) { + if (!it->is_regular_file(ec)) { + continue; + } + std::string extension = it->path().extension().string(); + std::transform(extension.begin(), extension.end(), extension.begin(), [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + if (extension == ".gguf" && gguf.empty()) { + gguf = it->path(); + } else if (extension == ".safetensors" && safetensors.empty()) { + safetensors = it->path(); + } + } + if (!gguf.empty()) { + return assets::open_tensor_source(gguf); + } + if (!safetensors.empty()) { + // Sharded safetensors ship an index beside the shards and are loaded + // through the package path instead (the index maps tensor names). + if (safetensors.extension() != ".safetensors" || + !engine::io::is_existing_file(safetensors.parent_path() / "model.safetensors.index.json")) { + return assets::open_tensor_source(safetensors); + } + } + } + // Sharded safetensors and other layouts stay on the package path. + return resources.open_tensor_source("weights"); +} + +/// R2T2 runs f32/f16/bf16/q8_0 weights. ggml's 4-bit and k-quant kernels are +/// not validated for this graph — a Q4_K checkpoint loads and then decodes to +/// an empty transcript — so reject lower precisions at load time with the fix +/// in the message instead of failing silently at inference time. +void validate_checkpoint_weight_types(const assets::TensorSource & source) { + static constexpr std::array kSupported = {"f32", "f16", "bf16", "q8_0"}; + for (const auto & meta : source.tensors()) { + if (meta.name.size() < 7 || meta.name.compare(meta.name.size() - 7, 7, ".weight") != 0) { + continue; + } + std::string dtype = meta.dtype; + std::transform(dtype.begin(), dtype.end(), dtype.begin(), [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + if (std::find(kSupported.begin(), kSupported.end(), dtype) != kSupported.end()) { + continue; + } + throw std::runtime_error( + "R2T2 ASR supports f32/f16/bf16/q8_0 weights, but tensor '" + meta.name + "' is " + meta.dtype + + ". Reconvert the checkpoint with --type q8_0 (or f16)."); + } +} + +std::shared_ptr make_assets( + assets::ResourceBundle resources) { + if (!resources.has_file("preprocessor_config") && + !resources.has_file("processor_config")) { + throw std::runtime_error( + "R2T2 ASR requires preprocessor_config.json or processor_config.json"); + } + const bool has_legacy_tokenizer = + resources.has_file("vocab") && resources.has_file("merges"); + if (!has_legacy_tokenizer && !resources.has_file("tokenizer_json")) { + throw std::runtime_error( + "R2T2 ASR requires vocab.json plus merges.txt, or tokenizer.json"); + } + R2T2ASRAssets assets; + assets.resources = std::move(resources); + assets.config = parse_config(assets.resources); + assets.model_weights = open_model_weights(assets.resources); + validate_checkpoint_weight_types(*assets.model_weights); + return std::make_shared(std::move(assets)); +} + +} // namespace + +std::shared_ptr load_r2t2_asr_assets(const std::filesystem::path & model_path) { + return load_r2t2_asr_assets(model_path, "r2t2_asr"); +} + +std::shared_ptr load_r2t2_asr_assets( + const std::filesystem::path & model_path, + std::string_view package_family) { + return make_assets(make_resource_bundle(model_path, package_family)); +} + +std::shared_ptr load_r2t2_asr_assets( + assets::ResourceBundle resources) { + return make_assets(std::move(resources)); +} + +} // namespace engine::community_models::r2t2_asr diff --git a/src/community_models/r2t2_asr/audio_encoder.cpp b/src/community_models/r2t2_asr/audio_encoder.cpp new file mode 100644 index 000000000..64141dba6 --- /dev/null +++ b/src/community_models/r2t2_asr/audio_encoder.cpp @@ -0,0 +1,615 @@ +#include "engine/community_models/r2t2_asr/audio_encoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/attention/scaled_dot_product_attention.h" +#include "engine/framework/modules/attention/transformer_blocks.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::r2t2_asr { + +namespace assets = engine::assets; +namespace modules = engine::modules; + +using Clock = std::chrono::steady_clock; + +constexpr size_t kDefaultAudioWeightContextBytes = 32ull * 1024ull * 1024ull; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +std::vector audio_chunk_lengths(int64_t input_frames, int64_t chunk_frames) { + if (input_frames <= 0 || chunk_frames <= 0) { + throw std::runtime_error("R2T2 ASR audio chunk lengths require positive sizes"); + } + std::vector chunks; + for (int64_t offset = 0; offset < input_frames; offset += chunk_frames) { + chunks.push_back(std::min(chunk_frames, input_frames - offset)); + } + return chunks; +} + +int64_t max_value(const std::vector & values) { + if (values.empty()) { + throw std::runtime_error("R2T2 ASR expected a non-empty length list"); + } + return *std::max_element(values.begin(), values.end()); +} + +int64_t sum_values(const std::vector & values) { + int64_t sum = 0; + for (const int64_t value : values) { + sum += value; + } + return sum; +} + +std::vector audio_attention_window_lengths(int64_t tokens, int64_t window_tokens) { + if (tokens <= 0 || window_tokens <= 0) { + throw std::runtime_error("R2T2 ASR audio attention window lengths require positive sizes"); + } + std::vector lengths; + for (int64_t offset = 0; offset < tokens; offset += window_tokens) { + lengths.push_back(std::min(window_tokens, tokens - offset)); + } + return lengths; +} + +std::vector audio_attention_mask(int64_t tokens, const std::vector & window_lengths) { + std::vector values(static_cast(tokens * tokens), -INFINITY); + int64_t offset = 0; + for (const int64_t length : window_lengths) { + for (int64_t row = 0; row < length; ++row) { + for (int64_t col = 0; col < length; ++col) { + values[static_cast((offset + row) * tokens + offset + col)] = 0.0F; + } + } + offset += length; + } + if (offset != tokens) { + throw std::runtime_error("R2T2 ASR audio attention windows do not cover all tokens"); + } + return values; +} + +std::vector sinusoidal_positions(int64_t length, int64_t channels) { + if (channels % 2 != 0) { + throw std::runtime_error("R2T2 ASR audio positional embedding requires even channel count"); + } + std::vector table(static_cast(length * channels), 0.0F); + const double increment = std::log(10000.0) / static_cast(channels / 2 - 1); + for (int64_t pos = 0; pos < length; ++pos) { + for (int64_t dim = 0; dim < channels / 2; ++dim) { + const double scaled = static_cast(pos) * std::exp(-increment * static_cast(dim)); + table[static_cast(pos * channels + dim)] = static_cast(std::sin(scaled)); + table[static_cast(pos * channels + channels / 2 + dim)] = static_cast(std::cos(scaled)); + } + } + return table; +} + +core::TensorValue reshape_audio_heads( + core::ModuleBuildContext & ctx, + const core::TensorValue & input, + int64_t heads, + int64_t dim) { + const auto contiguous = core::ensure_backend_addressable_layout(ctx, input); + return core::reshape_tensor( + ctx, + contiguous, + core::TensorShape::from_dims({input.shape.dims[0], input.shape.dims[1], heads, dim})); +} + +core::TensorValue audio_self_attention( + core::ModuleBuildContext & ctx, + const core::TensorValue & input, + const modules::AttentionWeights & weights, + int64_t hidden_size, + int64_t heads, + const core::TensorValue & attention_mask) { + const int64_t dim = hidden_size / heads; + const modules::LinearModule q_proj({hidden_size, hidden_size, true}); + const modules::LinearModule k_proj({hidden_size, hidden_size, true}); + const modules::LinearModule v_proj({hidden_size, hidden_size, true}); + const modules::LinearModule out_proj({hidden_size, hidden_size, true}); + + auto q = reshape_audio_heads(ctx, q_proj.build(ctx, input, {weights.q_weight, weights.q_bias}), heads, dim); + auto k = reshape_audio_heads(ctx, k_proj.build(ctx, input, {weights.k_weight, weights.k_bias}), heads, dim); + auto v = reshape_audio_heads(ctx, v_proj.build(ctx, input, {weights.v_weight, weights.v_bias}), heads, dim); + auto q_heads = modules::TransposeModule({{0, 2, 1, 3}, q.shape.rank}).build(ctx, q); + auto k_heads = modules::TransposeModule({{0, 2, 1, 3}, k.shape.rank}).build(ctx, k); + auto v_heads = modules::TransposeModule({{0, 2, 1, 3}, v.shape.rank}).build(ctx, v); + auto context = modules::ScaledDotProductAttentionModule({ + dim, + modules::ScaledDotProductAttentionLowering::Explicit, + GGML_PREC_F32, + }).build(ctx, q_heads, k_heads, v_heads, attention_mask); + context = core::ensure_backend_addressable_layout(ctx, context); + context = core::reshape_tensor(ctx, context, core::TensorShape::from_dims({input.shape.dims[0], input.shape.dims[1], hidden_size})); + return out_proj.build(ctx, context, {weights.out_weight, weights.out_bias}); +} + +core::TensorValue audio_encoder_layer( + core::ModuleBuildContext & ctx, + const core::TensorValue & input, + const modules::TransformerEncoderBlockWeights & weights, + const R2T2ASRAudioEncoderConfig & config, + const core::TensorValue & attention_mask) { + const modules::LayerNormModule norm({config.d_model, 1.0e-5F, true, true}); + auto attn_in = norm.build(ctx, input, weights.norm1); + auto attn = audio_self_attention( + ctx, + attn_in, + weights.self_attention, + config.d_model, + config.encoder_attention_heads, + attention_mask); + auto x = modules::AddModule().build(ctx, input, attn); + auto ff_in = norm.build(ctx, x, weights.norm2); + auto ff = modules::FeedForwardModule({config.d_model, config.encoder_ffn_dim, true, modules::GeluApproximation::ExactErf}) + .build(ctx, ff_in, weights.feed_forward); + return modules::AddModule().build(ctx, x, ff); +} + +struct Conv2dWeightsData { + core::TensorValue weight; + core::TensorValue bias; +}; + +struct LinearWeightsData { + core::TensorValue weight; + core::TensorValue bias; +}; + +struct AudioLayerWeights { + core::TensorValue self_attn_norm_weight; + core::TensorValue self_attn_norm_bias; + core::TensorValue q_proj_weight; + core::TensorValue q_proj_bias; + core::TensorValue k_proj_weight; + core::TensorValue k_proj_bias; + core::TensorValue v_proj_weight; + core::TensorValue v_proj_bias; + core::TensorValue out_proj_weight; + core::TensorValue out_proj_bias; + core::TensorValue final_norm_weight; + core::TensorValue final_norm_bias; + core::TensorValue fc1_weight; + core::TensorValue fc1_bias; + core::TensorValue fc2_weight; + core::TensorValue fc2_bias; +}; + +struct R2T2ASRAudioEncoderWeights { + std::shared_ptr store; + Conv2dWeightsData conv1; + Conv2dWeightsData conv2; + Conv2dWeightsData conv3; + core::TensorValue conv_out_weight; + std::vector layers; + core::TensorValue ln_post_weight; + core::TensorValue ln_post_bias; + LinearWeightsData proj1; + LinearWeightsData proj2; + core::TensorValue positional_embedding; +}; + +Conv2dWeightsData load_conv2d( + core::BackendWeightStore & store, + const assets::TensorSource & source, + const std::string & prefix, + assets::TensorStorageType storage_type, + std::initializer_list weight_shape, + int64_t out_channels) { + return { + store.load_tensor(source, prefix + ".weight", storage_type, weight_shape), + store.load_f32_tensor(source, prefix + ".bias", {out_channels}), + }; +} + +std::shared_ptr load_weights( + const R2T2ASRAssets & assets, + ggml_backend_t backend, + core::BackendType backend_type, + assets::TensorStorageType storage_type) { + const auto & config = assets.config.audio_encoder; + const auto & source = *assets.model_weights; + const std::string audio_prefix = assets.config.hf_transformers_layout + ? "model.audio_tower" + : "thinker.audio_tower"; + const std::string projector_prefix = assets.config.hf_transformers_layout + ? "model.multi_modal_projector" + : audio_prefix; + auto weights = std::make_shared(); + auto store = std::make_shared( + backend, + backend_type, + "r2t2_asr.audio_encoder.weights", + kDefaultAudioWeightContextBytes); + weights->store = store; + weights->conv1 = load_conv2d( + *store, + source, + audio_prefix + ".conv2d1", + storage_type, + {config.downsample_hidden_size, 1, 3, 3}, + config.downsample_hidden_size); + weights->conv2 = load_conv2d( + *store, + source, + audio_prefix + ".conv2d2", + storage_type, + {config.downsample_hidden_size, config.downsample_hidden_size, 3, 3}, + config.downsample_hidden_size); + weights->conv3 = load_conv2d( + *store, + source, + audio_prefix + ".conv2d3", + storage_type, + {config.downsample_hidden_size, config.downsample_hidden_size, 3, 3}, + config.downsample_hidden_size); + const int64_t conv_freq = (((config.num_mel_bins + 1) / 2 + 1) / 2 + 1) / 2; + weights->conv_out_weight = store->load_tensor( + source, + audio_prefix + ".conv_out.weight", + storage_type, + {config.d_model, config.downsample_hidden_size * conv_freq}); + weights->layers.reserve(static_cast(config.encoder_layers)); + for (int64_t layer = 0; layer < config.encoder_layers; ++layer) { + const std::string prefix = audio_prefix + ".layers." + std::to_string(layer); + AudioLayerWeights w; + w.self_attn_norm_weight = store->load_f32_tensor(source, prefix + ".self_attn_layer_norm.weight", {config.d_model}); + w.self_attn_norm_bias = store->load_f32_tensor(source, prefix + ".self_attn_layer_norm.bias", {config.d_model}); + w.q_proj_weight = store->load_tensor(source, prefix + ".self_attn.q_proj.weight", storage_type, {config.d_model, config.d_model}); + w.q_proj_bias = store->load_f32_tensor(source, prefix + ".self_attn.q_proj.bias", {config.d_model}); + w.k_proj_weight = store->load_tensor(source, prefix + ".self_attn.k_proj.weight", storage_type, {config.d_model, config.d_model}); + w.k_proj_bias = store->load_f32_tensor(source, prefix + ".self_attn.k_proj.bias", {config.d_model}); + w.v_proj_weight = store->load_tensor(source, prefix + ".self_attn.v_proj.weight", storage_type, {config.d_model, config.d_model}); + w.v_proj_bias = store->load_f32_tensor(source, prefix + ".self_attn.v_proj.bias", {config.d_model}); + w.out_proj_weight = store->load_tensor(source, prefix + ".self_attn.out_proj.weight", storage_type, {config.d_model, config.d_model}); + w.out_proj_bias = store->load_f32_tensor(source, prefix + ".self_attn.out_proj.bias", {config.d_model}); + w.final_norm_weight = store->load_f32_tensor(source, prefix + ".final_layer_norm.weight", {config.d_model}); + w.final_norm_bias = store->load_f32_tensor(source, prefix + ".final_layer_norm.bias", {config.d_model}); + w.fc1_weight = store->load_tensor(source, prefix + ".fc1.weight", storage_type, {config.encoder_ffn_dim, config.d_model}); + w.fc1_bias = store->load_f32_tensor(source, prefix + ".fc1.bias", {config.encoder_ffn_dim}); + w.fc2_weight = store->load_tensor(source, prefix + ".fc2.weight", storage_type, {config.d_model, config.encoder_ffn_dim}); + w.fc2_bias = store->load_f32_tensor(source, prefix + ".fc2.bias", {config.d_model}); + weights->layers.push_back(std::move(w)); + } + weights->ln_post_weight = store->load_f32_tensor(source, audio_prefix + ".ln_post.weight", {config.d_model}); + weights->ln_post_bias = store->load_f32_tensor(source, audio_prefix + ".ln_post.bias", {config.d_model}); + const std::string proj1 = assets.config.hf_transformers_layout ? ".linear_1" : ".proj1"; + const std::string proj2 = assets.config.hf_transformers_layout ? ".linear_2" : ".proj2"; + weights->proj1 = { + store->load_tensor(source, projector_prefix + proj1 + ".weight", storage_type, {config.d_model, config.d_model}), + store->load_f32_tensor(source, projector_prefix + proj1 + ".bias", {config.d_model}), + }; + weights->proj2 = { + store->load_tensor(source, projector_prefix + proj2 + ".weight", storage_type, {config.output_dim, config.d_model}), + store->load_f32_tensor(source, projector_prefix + proj2 + ".bias", {config.output_dim}), + }; + weights->positional_embedding = store->make_f32( + core::TensorShape::from_dims({config.max_source_positions, config.d_model}), + sinusoidal_positions(config.max_source_positions, config.d_model)); + store->upload(); + return weights; +} + +modules::TransformerEncoderBlockWeights bind_layer(const AudioLayerWeights & weights) { + modules::TransformerEncoderBlockWeights block; + block.norm1 = {weights.self_attn_norm_weight, weights.self_attn_norm_bias}; + block.self_attention.q_weight = weights.q_proj_weight; + block.self_attention.q_bias = weights.q_proj_bias; + block.self_attention.k_weight = weights.k_proj_weight; + block.self_attention.k_bias = weights.k_proj_bias; + block.self_attention.v_weight = weights.v_proj_weight; + block.self_attention.v_bias = weights.v_proj_bias; + block.self_attention.qkv_weight = std::nullopt; + block.self_attention.qkv_bias = std::nullopt; + block.self_attention.out_weight = weights.out_proj_weight; + block.self_attention.out_bias = weights.out_proj_bias; + block.layer_scale1 = std::nullopt; + block.norm2 = {weights.final_norm_weight, weights.final_norm_bias}; + block.feed_forward.fc1_weight = weights.fc1_weight; + block.feed_forward.fc1_bias = weights.fc1_bias; + block.feed_forward.fc2_weight = weights.fc2_weight; + block.feed_forward.fc2_bias = weights.fc2_bias; + block.layer_scale2 = std::nullopt; + return block; +} + +struct GgmlGallocrDeleter { + void operator()(ggml_gallocr_t alloc) const noexcept { + if (alloc != nullptr) { + ggml_gallocr_free(alloc); + } + } +}; + +class R2T2ASRAudioEncoderGraph { +public: + R2T2ASRAudioEncoderGraph( + std::shared_ptr assets, + std::shared_ptr weights, + core::ExecutionContext & execution, + size_t graph_arena_bytes, + int64_t frames) + : assets_(std::move(assets)), + weights_(std::move(weights)), + backend_(execution.backend()), + backend_type_(execution.backend_type()), + compute_threads_(std::max(1, execution.config().threads)), + frames_(frames) { + if (assets_ == nullptr || weights_ == nullptr) { + throw std::runtime_error("R2T2 ASR audio encoder graph requires assets and weights"); + } + if (backend_ == nullptr) { + throw std::runtime_error("R2T2 ASR audio encoder backend is not initialized"); + } + if (frames_ <= 0) { + throw std::runtime_error("R2T2 ASR audio encoder graph requires positive frame count"); + } + const auto build_start = Clock::now(); + const auto & config = assets_->config.audio_encoder; + chunk_frame_limit_ = config.n_window * 2; + chunk_lengths_ = audio_chunk_lengths(frames_, chunk_frame_limit_); + chunk_frames_ = max_value(chunk_lengths_); + chunk_count_ = static_cast(chunk_lengths_.size()); + chunk_token_lengths_.reserve(chunk_lengths_.size()); + for (const int64_t chunk_length : chunk_lengths_) { + chunk_token_lengths_.push_back(r2t2_asr_audio_encoder_token_count(chunk_length)); + } + output_tokens_ = sum_values(chunk_token_lengths_); + const int64_t max_chunk_tokens = max_value(chunk_token_lengths_); + attention_window_tokens_ = max_chunk_tokens * (config.n_window_infer / chunk_frame_limit_); + if (output_tokens_ > config.max_source_positions) { + throw std::runtime_error("R2T2 ASR audio encoder token count exceeds max_source_positions"); + } + attention_window_lengths_ = audio_attention_window_lengths(output_tokens_, attention_window_tokens_); + ggml_init_params params{graph_arena_bytes, nullptr, true}; + ctx_.reset(ggml_init(params)); + if (ctx_ == nullptr) { + throw std::runtime_error("failed to initialize R2T2 ASR audio encoder graph context"); + } + + core::ModuleBuildContext ctx{ctx_.get(), "r2t2_asr.audio_encoder", backend_type_}; + auto input = core::make_tensor( + ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({chunk_count_, config.num_mel_bins, chunk_frames_})); + input_ = input.tensor; + auto x = core::reshape_tensor( + ctx, + input, + core::TensorShape::from_dims({chunk_count_, 1, config.num_mel_bins, chunk_frames_})); + x = modules::Conv2dModule({1, config.downsample_hidden_size, 3, 3, 2, 2, 1, 1, 1, 1, true}) + .build(ctx, x, {weights_->conv1.weight, weights_->conv1.bias}); + x = modules::GeluModule().build(ctx, x); + x = modules::Conv2dModule( + {config.downsample_hidden_size, config.downsample_hidden_size, 3, 3, 2, 2, 1, 1, 1, 1, true}) + .build(ctx, x, {weights_->conv2.weight, weights_->conv2.bias}); + x = modules::GeluModule().build(ctx, x); + x = modules::Conv2dModule( + {config.downsample_hidden_size, config.downsample_hidden_size, 3, 3, 2, 2, 1, 1, 1, 1, true}) + .build(ctx, x, {weights_->conv3.weight, weights_->conv3.bias}); + x = modules::GeluModule().build(ctx, x); + // Keep the ggml layout aligned with the Python conv output before flattening [B, T, C, F]. + // The permutation is a view; the framework layout helper turns it into a + // backend-addressable tensor so no raw ggml_cont* call is needed here. + const auto transposed_shape = core::TensorShape::from_dims({x.shape.dims[0], x.shape.dims[3], x.shape.dims[1], x.shape.dims[2]}); + x = core::wrap_tensor(ggml_permute(ctx.ggml, x.tensor, 2, 0, 1, 3), transposed_shape, x.type); + x = core::ensure_backend_addressable_layout(ctx, x); + x = core::reshape_tensor( + ctx, + x, + core::TensorShape::from_dims({chunk_count_, x.shape.dims[1], x.shape.dims[2] * x.shape.dims[3]})); + x = modules::LinearModule({x.shape.last_dim(), config.d_model, false}).build( + ctx, + x, + {weights_->conv_out_weight, std::nullopt}); + const int64_t tokens = x.shape.dims[1]; + auto pos = weights_->positional_embedding; + pos = modules::SliceModule({0, 0, tokens}).build(ctx, pos); + pos = core::reshape_tensor(ctx, pos, core::TensorShape::from_dims({1, tokens, config.d_model})); + if (chunk_count_ > 1) { + pos = modules::RepeatModule({core::TensorShape::from_dims({chunk_count_, tokens, config.d_model})}) + .build(ctx, pos); + } + x = modules::AddModule().build(ctx, x, pos); + core::TensorValue compacted; + for (int64_t chunk = 0; chunk < chunk_count_; ++chunk) { + auto chunk_value = modules::SliceModule({0, chunk, 1}).build(ctx, x); + const int64_t valid_tokens = chunk_token_lengths_[static_cast(chunk)]; + if (valid_tokens < tokens) { + chunk_value = modules::SliceModule({1, 0, valid_tokens}).build(ctx, chunk_value); + } + compacted = compacted.valid() + ? modules::ConcatModule({1}).build(ctx, compacted, chunk_value) + : chunk_value; + } + x = compacted; + attention_mask_values_ = audio_attention_mask(output_tokens_, attention_window_lengths_); + auto attention_mask = core::make_tensor( + ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, 1, output_tokens_, output_tokens_})); + attention_mask_ = attention_mask.tensor; + for (const auto & layer : weights_->layers) { + x = audio_encoder_layer(ctx, x, bind_layer(layer), config, attention_mask); + } + x = modules::LayerNormModule({config.d_model, 1.0e-5F, true, true}) + .build(ctx, x, {weights_->ln_post_weight, weights_->ln_post_bias}); + x = modules::LinearModule({config.d_model, config.d_model, true}).build( + ctx, + x, + {weights_->proj1.weight, weights_->proj1.bias}); + x = modules::GeluModule().build(ctx, x); + x = modules::LinearModule({config.d_model, config.output_dim, true}).build( + ctx, + x, + {weights_->proj2.weight, weights_->proj2.bias}); + output_ = x.tensor; + output_tokens_ = x.shape.dims[1]; + output_dim_ = x.shape.dims[2]; + ggml_set_output(output_); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + ggml_build_forward_expand(graph_, output_); + // unique_ptr so a throw below frees the partial reservation (a + // throwing constructor runs no destructor) + const auto try_alloc = [&]() { + gallocr_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_))); + return gallocr_ != nullptr && ggml_gallocr_alloc_graph(gallocr_.get(), graph_); + }; + if (!try_alloc() && + (engine::core::trim_backend_pools(backend_), !try_alloc())) { + throw std::runtime_error("failed to allocate R2T2 ASR audio encoder graph"); + } + ggml_backend_tensor_set(attention_mask_, attention_mask_values_.data(), 0, attention_mask_values_.size() * sizeof(float)); + debug::timing_log_scalar("r2t2_asr.audio_encoder.graph.build_ms", engine::debug::elapsed_ms(build_start, Clock::now())); + debug::trace_log_scalar("r2t2_asr.audio_encoder.frames", frames_); + } + + ~R2T2ASRAudioEncoderGraph() { + engine::core::release_backend_graph_resources(backend_, graph_, true); + } + + bool matches(const R2T2ASRAudioEncoderWeights & weights, int64_t frames, ggml_backend_t backend, int threads) const { + return weights_.get() == &weights && frames_ == frames && backend_ == backend && compute_threads_ == std::max(1, threads); + } + + R2T2ASRAudioEmbeddings run(const R2T2ASRAudioFeatures & features) { + const auto & config = assets_->config.audio_encoder; + if (features.mel_bins != config.num_mel_bins || features.frames != frames_) { + throw std::runtime_error("R2T2 ASR audio encoder feature shape mismatch"); + } + if (static_cast(features.values.size()) != config.num_mel_bins * frames_) { + throw std::runtime_error("R2T2 ASR audio encoder feature value count mismatch"); + } + std::vector padded_features(static_cast(chunk_count_ * config.num_mel_bins * chunk_frames_), 0.0F); + int64_t source_frame = 0; + for (int64_t chunk = 0; chunk < chunk_count_; ++chunk) { + const int64_t chunk_length = chunk_lengths_[static_cast(chunk)]; + for (int64_t mel = 0; mel < config.num_mel_bins; ++mel) { + const size_t dst = static_cast((chunk * config.num_mel_bins + mel) * chunk_frames_); + const size_t src = static_cast(mel * frames_ + source_frame); + std::copy_n( + features.values.begin() + static_cast(src), + static_cast(chunk_length), + padded_features.begin() + static_cast(dst)); + } + source_frame += chunk_length; + } + auto timing_start = Clock::now(); + ggml_backend_tensor_set(input_, padded_features.data(), 0, padded_features.size() * sizeof(float)); + ggml_backend_tensor_set(attention_mask_, attention_mask_values_.data(), 0, attention_mask_values_.size() * sizeof(float)); + debug::timing_log_scalar("r2t2_asr.audio_encoder.input_upload_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); + core::set_backend_threads(backend_, compute_threads_); + timing_start = Clock::now(); + const ggml_status status = engine::core::compute_backend_graph(backend_, graph_); + ggml_backend_synchronize(backend_); + debug::timing_log_scalar("r2t2_asr.audio_encoder.graph.compute_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("R2T2 ASR audio encoder graph compute failed"); + } + R2T2ASRAudioEmbeddings out; + out.tokens = output_tokens_; + out.hidden_size = output_dim_; + out.values.resize(static_cast(out.tokens * out.hidden_size)); + timing_start = Clock::now(); + ggml_backend_tensor_get(output_, out.values.data(), 0, out.values.size() * sizeof(float)); + debug::timing_log_scalar("r2t2_asr.audio_encoder.output_read_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); + return out; + } + +private: + std::shared_ptr assets_; + std::shared_ptr weights_; + ggml_backend_t backend_ = nullptr; + core::BackendType backend_type_ = core::BackendType::Cpu; + int compute_threads_ = 1; + int64_t frames_ = 0; + int64_t chunk_frame_limit_ = 0; + int64_t chunk_frames_ = 0; + int64_t chunk_count_ = 0; + std::vector chunk_lengths_; + std::vector chunk_token_lengths_; + std::vector attention_window_lengths_; + std::vector attention_mask_values_; + int64_t attention_window_tokens_ = 0; + int64_t output_tokens_ = 0; + int64_t output_dim_ = 0; + std::unique_ptr ctx_; + ggml_tensor * input_ = nullptr; + ggml_tensor * attention_mask_ = nullptr; + ggml_tensor * output_ = nullptr; + ggml_cgraph * graph_ = nullptr; + std::unique_ptr, GgmlGallocrDeleter> gallocr_; +}; + +R2T2ASRAudioEncoderRuntime::R2T2ASRAudioEncoderRuntime( + std::shared_ptr assets, + core::ExecutionContext & execution, + size_t graph_arena_bytes, + assets::TensorStorageType weight_storage_type) + : assets_(std::move(assets)), + execution_(&execution), + graph_arena_bytes_(graph_arena_bytes) { + if (assets_ == nullptr) { + throw std::runtime_error("R2T2 ASR audio encoder requires assets"); + } + if (graph_arena_bytes_ == 0) { + throw std::runtime_error("R2T2 ASR audio encoder graph arena must be non-zero"); + } + weights_ = load_weights(*assets_, execution.backend(), execution.backend_type(), weight_storage_type); +} + +R2T2ASRAudioEncoderRuntime::~R2T2ASRAudioEncoderRuntime() = default; + +R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeatures & features) { + if (execution_ == nullptr) { + throw std::runtime_error("R2T2 ASR audio encoder execution context is null"); + } + if (features.encoder_tokens != r2t2_asr_audio_encoder_token_count(features.frames)) { + throw std::runtime_error("R2T2 ASR audio encoder token count mismatch"); + } + const int threads = std::max(1, execution_->config().threads); + if (graph_ == nullptr || !graph_->matches(*weights_, features.frames, execution_->backend(), threads)) { + graph_.reset(); + graph_ = std::make_unique( + assets_, + weights_, + *execution_, + graph_arena_bytes_, + features.frames); + } else { + debug::timing_log_scalar("r2t2_asr.audio_encoder.graph.build_ms", 0.0); + debug::trace_log_scalar("r2t2_asr.audio_encoder.frames", features.frames); + } + auto out = graph_->run(features); + if (out.tokens != features.encoder_tokens) { + throw std::runtime_error("R2T2 ASR audio encoder output token count mismatch"); + } + return out; +} + +} // namespace engine::community_models::r2t2_asr diff --git a/src/community_models/r2t2_asr/frontend_whisper.cpp b/src/community_models/r2t2_asr/frontend_whisper.cpp new file mode 100644 index 000000000..8cbcf60a0 --- /dev/null +++ b/src/community_models/r2t2_asr/frontend_whisper.cpp @@ -0,0 +1,93 @@ +#include "engine/community_models/r2t2_asr/frontend_whisper.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/dsp.h" +#include "engine/framework/audio/waveform_ops.h" +#include "engine/framework/debug/profiler.h" + +#include +#include +#include +#include +#include + +namespace engine::community_models::r2t2_asr { +namespace { + +constexpr int kMinInputSamples = 8000; +using Clock = std::chrono::steady_clock; + +void validate_audio_input(const runtime::AudioBuffer & audio) { + if (audio.sample_rate <= 0) { + throw std::runtime_error("R2T2 ASR audio sample_rate must be positive"); + } + if (audio.channels <= 0) { + throw std::runtime_error("R2T2 ASR audio channels must be positive"); + } + if (audio.samples.empty()) { + throw std::runtime_error("R2T2 ASR audio is empty"); + } + if (audio.samples.size() % static_cast(audio.channels) != 0) { + throw std::runtime_error("R2T2 ASR interleaved audio size is not divisible by channel count"); + } +} + +std::vector normalize_audio(const runtime::AudioBuffer & audio, int sample_rate) { + validate_audio_input(audio); + auto mono = engine::audio::convert_interleaved_audio_to_mono_linear_resampled( + audio.samples, + audio.sample_rate, + audio.channels, + sample_rate); + engine::audio::normalize_peak_to_unit_range_and_clamp_in_place(mono); + if (mono.size() < kMinInputSamples) { + mono.resize(kMinInputSamples, 0.0F); + } + return mono; +} + +engine::audio::WhisperLogMelConfig make_extractor_config(const std::shared_ptr & assets) { + if (assets == nullptr) { + throw std::runtime_error("R2T2 ASR Whisper frontend requires assets"); + } + const auto & config = assets->config.frontend; + return { + config.sample_rate, + config.n_fft, + config.hop_length, + config.feature_size, + engine::audio::STFTFamily::Default, + }; +} + +} // namespace + +R2T2ASRWhisperFrontend::R2T2ASRWhisperFrontend(std::shared_ptr assets) + : assets_(std::move(assets)), + extractor_(make_extractor_config(assets_)) {} + +R2T2ASRAudioFeatures R2T2ASRWhisperFrontend::extract(const runtime::AudioBuffer & audio) const { + const auto normalize_start = Clock::now(); + const auto & config = assets_->config.frontend; + if (config.sample_rate <= 0 || config.feature_size <= 0 || config.hop_length <= 0 || config.n_fft <= 0) { + throw std::runtime_error("R2T2 ASR Whisper frontend config is invalid"); + } + auto samples = normalize_audio(audio, config.sample_rate); + const auto normalize_end = Clock::now(); + + const auto feature_start = Clock::now(); + auto features = extractor_.compute(samples); + const auto feature_end = Clock::now(); + + R2T2ASRAudioFeatures out; + out.values = std::move(features.values); + out.attention_mask.assign(static_cast(features.frames), 1); + out.mel_bins = features.mel_bins; + out.frames = features.frames; + out.encoder_tokens = r2t2_asr_audio_encoder_token_count(out.frames); + debug::timing_log_scalar("r2t2_asr.frontend.normalize_ms", engine::debug::elapsed_ms(normalize_start, normalize_end)); + debug::timing_log_scalar("r2t2_asr.frontend.log_mel_ms", engine::debug::elapsed_ms(feature_start, feature_end)); + return out; +} + +} // namespace engine::community_models::r2t2_asr diff --git a/src/community_models/r2t2_asr/prompt_asr.cpp b/src/community_models/r2t2_asr/prompt_asr.cpp new file mode 100644 index 000000000..e82863893 --- /dev/null +++ b/src/community_models/r2t2_asr/prompt_asr.cpp @@ -0,0 +1,12 @@ +#include "engine/community_models/r2t2_asr/prompt_asr.h" + +namespace engine::community_models::r2t2_asr { + +R2T2ASRPromptBuilder::R2T2ASRPromptBuilder(const R2T2ASRTextTokenizer & tokenizer) + : tokenizer_(tokenizer) {} + +R2T2ASRPrompt R2T2ASRPromptBuilder::build(const R2T2ASRRequest & request, int64_t audio_feature_tokens) const { + return tokenizer_.build_prompt(request.context, request.language, audio_feature_tokens); +} + +} // namespace engine::community_models::r2t2_asr diff --git a/src/community_models/r2t2_asr/session.cpp b/src/community_models/r2t2_asr/session.cpp new file mode 100644 index 000000000..88d031b37 --- /dev/null +++ b/src/community_models/r2t2_asr/session.cpp @@ -0,0 +1,602 @@ +#include "engine/community_models/r2t2_asr/session.h" + +#include "engine/framework/audio/chunking.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" +#include "engine/community_models/r2t2_asr/text_postprocess.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::r2t2_asr { +namespace { + +using Clock = std::chrono::steady_clock; + +constexpr double kOfflineChunkSeconds = 30.0; + +std::shared_ptr require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("R2T2 ASR session requires assets"); + } + return assets; +} + +void validate_matmul_weight_storage(engine::assets::TensorStorageType storage_type, const char * option_name) { + if (storage_type == engine::assets::TensorStorageType::Native || + storage_type == engine::assets::TensorStorageType::F32 || + storage_type == engine::assets::TensorStorageType::F16 || + storage_type == engine::assets::TensorStorageType::BF16 || + storage_type == engine::assets::TensorStorageType::Q8_0) { + return; + } + throw std::runtime_error(std::string(option_name) + " currently supports only native, f32, f16, bf16, and q8_0"); +} + +void validate_audio_encoder_weight_storage(engine::assets::TensorStorageType storage_type) { + if (storage_type == engine::assets::TensorStorageType::Native || + storage_type == engine::assets::TensorStorageType::F32 || + storage_type == engine::assets::TensorStorageType::F16) { + return; + } + throw std::runtime_error("r2t2_asr.audio_encoder_weight_type currently supports only native, f32, and f16"); +} + +engine::assets::TensorStorageType option_weight_type( + const runtime::SessionOptions & options, + const char * key, + engine::assets::TensorStorageType default_value) { + const auto it = options.options.find(key); + if (it == options.options.end()) { + return default_value; + } + return engine::assets::parse_tensor_storage_type(it->second); +} + +int64_t audio_frame_count(const runtime::AudioBuffer & audio) { + if (audio.channels <= 0) { + throw std::runtime_error("R2T2 ASR audio requires positive channel count"); + } + if (audio.samples.size() % static_cast(audio.channels) != 0) { + throw std::runtime_error("R2T2 ASR audio samples must be divisible by channel count"); + } + return static_cast(audio.samples.size() / static_cast(audio.channels)); +} + +bool language_is_supported(const R2T2ASRAssets & assets, const std::string & language) { + const auto & supported = assets.config.supported_languages; + return std::find(supported.begin(), supported.end(), language) != supported.end(); +} + +} // namespace + +R2T2ASRSession::R2T2ASRSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets) + : RuntimeSessionBase(options), + task_(task), + assets_(require_assets(std::move(assets))), + audio_encoder_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.audio_encoder_graph_arena_mb"}, 128ull * 1024ull * 1024ull)), + thinker_prefill_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.thinker_prefill_graph_arena_mb"}, 256ull * 1024ull * 1024ull)), + thinker_decode_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.thinker_decode_graph_arena_mb"}, 256ull * 1024ull * 1024ull)), + thinker_weight_context_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.thinker_weight_context_mb"}, 64ull * 1024ull * 1024ull)), + audio_encoder_weight_storage_type_(option_weight_type(options, "r2t2_asr.audio_encoder_weight_type", engine::assets::TensorStorageType::Native)), + thinker_weight_storage_type_(option_weight_type( + options, + "r2t2_asr.thinker_weight_type", + option_weight_type(options, "r2t2_asr.weight_type", engine::assets::TensorStorageType::Native))), + tokenizer_(assets_), + frontend_(assets_), + audio_encoder_(assets_, execution_context(), audio_encoder_graph_arena_bytes_, audio_encoder_weight_storage_type_), + thinker_( + assets_, + execution_context(), + thinker_prefill_graph_arena_bytes_, + thinker_decode_graph_arena_bytes_, + thinker_weight_context_bytes_, + thinker_weight_storage_type_) { + if (task_.task != runtime::VoiceTaskKind::Asr) { + throw std::runtime_error("R2T2 ASR only supports VoiceTaskKind::Asr"); + } + if (task_.mode != runtime::RunMode::Offline && task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error("R2T2 ASR supports offline and streaming sessions"); + } + validate_audio_encoder_weight_storage(audio_encoder_weight_storage_type_); + validate_matmul_weight_storage(thinker_weight_storage_type_, "r2t2_asr.thinker_weight_type"); + + if (const auto value = runtime::parse_float_option(options.options, {"r2t2_asr.chunk_size_ms"})) { + stream_config_.chunk_seconds = static_cast(*value) / 1000.0; + } + if (const auto value = runtime::parse_int_option(options.options, {"r2t2_asr.unfixed_chunk_num"})) { + stream_config_.unfixed_chunk_num = *value; + } + if (const auto value = runtime::parse_int_option(options.options, {"r2t2_asr.unfixed_token_num"})) { + stream_config_.unfixed_token_num = *value; + } + if (const auto value = runtime::find_option(options.options, {"r2t2_asr.rollback_punctuation"})) { + stream_config_.rollback_punctuation = runtime::parse_bool_option(*value, "r2t2_asr.rollback_punctuation"); + } + if (const auto value = runtime::parse_int_option(options.options, {"r2t2_asr.max_tokens"})) { + stream_config_.max_new_tokens = *value; + } + if (stream_config_.chunk_seconds <= 0.0) { + throw std::runtime_error("r2t2_asr.chunk_size_ms must be positive"); + } + if (stream_config_.unfixed_chunk_num < 0 || stream_config_.unfixed_token_num < 0) { + throw std::runtime_error("r2t2_asr.unfixed_chunk_num and r2t2_asr.unfixed_token_num must be non-negative"); + } + if (stream_config_.max_new_tokens <= 0) { + throw std::runtime_error("r2t2_asr.max_tokens must be positive"); + } + for (const auto & [key, value] : options.options) { + (void) value; + if (key.rfind("r2t2_asr.", 0) == 0 && + key != "r2t2_asr.audio_encoder_graph_arena_mb" && + key != "r2t2_asr.thinker_prefill_graph_arena_mb" && + key != "r2t2_asr.thinker_decode_graph_arena_mb" && + key != "r2t2_asr.thinker_weight_context_mb" && + key != "r2t2_asr.audio_encoder_weight_type" && + key != "r2t2_asr.thinker_weight_type" && + key != "r2t2_asr.weight_type" && + key != "r2t2_asr.chunk_size_ms" && + key != "r2t2_asr.unfixed_chunk_num" && + key != "r2t2_asr.unfixed_token_num" && + key != "r2t2_asr.rollback_punctuation" && + key != "r2t2_asr.max_tokens") { + throw std::runtime_error("unknown R2T2 ASR session option: " + key); + } + } + assets_->model_weights->release_storage(); +} + +R2T2ASRSession::~R2T2ASRSession() = default; + +std::string R2T2ASRSession::family() const { + return "r2t2_asr"; +} + +runtime::VoiceTaskKind R2T2ASRSession::task_kind() const { + return task_.task; +} + +runtime::RunMode R2T2ASRSession::run_mode() const { + return task_.mode; +} + +void R2T2ASRSession::prepare(const runtime::SessionPreparationRequest & request) { + (void) request; + mark_prepared(); +} + +R2T2ASRRequest R2T2ASRSession::make_request(const runtime::TaskRequest & request) const { + if (!request.audio_input.has_value()) { + throw std::runtime_error("R2T2 ASR run() requires audio_input"); + } + R2T2ASRRequest out; + out.audio = *request.audio_input; + out.generation.max_new_tokens = assets_->config.max_new_tokens; + if (request.text_input.has_value()) { + out.context = request.text_input->text; + out.language = request.text_input->language; + } + if (const auto value = runtime::find_option(request.options, {"language"})) { + out.language = *value == "Auto" ? std::string() : *value; + } + if (const auto value = runtime::parse_int_option(request.options, {"max_tokens"})) { + out.generation.max_new_tokens = *value; + if (out.generation.max_new_tokens <= 0) { + throw std::runtime_error("R2T2 ASR max_tokens must be positive"); + } + } + if (!out.language.empty()) { + out.language = resolve_language(out.language); + if (!language_is_supported(*assets_, out.language)) { + throw std::runtime_error("R2T2 ASR language is not supported by this model: " + out.language); + } + } + return out; +} + +R2T2ASRResult R2T2ASRSession::run_single(const R2T2ASRRequest & request) { + const auto wall_start = Clock::now(); + const auto features = frontend_.extract(request.audio); + const auto prompt = tokenizer_.build_prompt(request.context, request.language, features.encoder_tokens); + const auto audio_embeddings = audio_encoder_.encode(features); + const auto tokens = thinker_.generate(prompt, audio_embeddings, request.generation); + const std::string raw = tokenizer_.decode(tokens.token_ids); + + R2T2ASRResult result; + const auto parsed = parse_asr_output(raw, request.language); + result.language = parsed.language.empty() ? request.language : parsed.language; + result.text = truncate_at_pipe(parsed.text); + debug::timing_log_scalar("r2t2_asr.single_ms", engine::debug::elapsed_ms(wall_start)); + debug::trace_log_scalar("r2t2_asr.audio_frames", features.frames); + return result; +} + +runtime::TaskResult R2T2ASRSession::run(const runtime::TaskRequest & request) { + require_prepared("R2T2 ASR run()"); + if (task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("R2T2 ASR run() requires an offline session"); + } + const auto & audio = make_request(request).audio; + const int64_t frames = audio_frame_count(audio); + const int64_t frames_per_chunk = std::max( + 1, + static_cast(std::llround(kOfflineChunkSeconds * static_cast(audio.sample_rate)))); + // Use the framework chunk planner so padding and tail alignment stay + // consistent with the other ASR families instead of hand-rolling slices. + const auto chunk_spans = engine::audio::plan_audio_chunks( + frames, + engine::audio::AudioChunkSpec{ + frames_per_chunk, + frames_per_chunk, + engine::audio::AudioChunkPadMode::Zero, + engine::audio::AudioChunkTailAlignment::Start, + 0, + }); + runtime::TaskResult merged; + std::ostringstream text; + for (const auto & span : chunk_spans) { + const auto valid_span = runtime::TimeSpan{span.output_start_sample, span.output_start_sample + span.valid_samples}; + runtime::TaskRequest item_request = request; + item_request.audio_input = engine::audio::slice_audio_buffer(audio, valid_span); + auto item = run_single(make_request(item_request)); + if (!item.text.empty()) { + if (text.tellp() > 0) { + text << ' '; + } + text << item.text; + } + if (!item.language.empty()) { + if (merged.text_output == std::nullopt) { + merged.text_output = runtime::Transcript{"", item.language}; + } else if (merged.text_output->language.empty()) { + merged.text_output->language = item.language; + } + } + } + if (merged.text_output == std::nullopt) { + merged.text_output = runtime::Transcript{"", ""}; + } + merged.text_output->text = text.str(); + return merged; +} + +std::string R2T2ASRSession::generate_text( + const R2T2ASRPrompt & prompt, + const R2T2ASRAudioEmbeddings & embeddings) { + R2T2ASRGenerationOptions options; + options.max_new_tokens = stream_config_.max_new_tokens; + const auto tokens = thinker_.generate(prompt, embeddings, options); + return tokenizer_.decode(tokens.token_ids); +} + +std::string R2T2ASRSession::decode_rollback_prefix( + const std::vector & ids, + int64_t rollback) const { + // Mirrors the reference U+FFFD rollback loop: grow the rollback until the + // decoded prefix contains no replacement character. + int64_t k = rollback; + while (true) { + const int64_t end_index = std::max(0, static_cast(ids.size()) - k); + std::string prefix; + if (end_index > 0) { + prefix = sanitize_utf8_lossy(tokenizer_.decode(std::vector(ids.begin(), ids.begin() + static_cast(end_index)))); + } + if (prefix.find(kReplacementChar) == std::string::npos) { + return prefix; + } + if (end_index == 0) { + return {}; + } + ++k; + } +} + +std::string R2T2ASRSession::build_stream_prefix(bool final_flush) const { + if (chunk_id_ < stream_config_.unfixed_chunk_num) { + return {}; + } + const std::string raw_truncated = truncate_at_pipe(raw_decoded_); + const auto ids = tokenizer_.encode(raw_truncated); + if (final_flush) { + // finish_streaming_transcribe uses a fixed rollback without the + // replacement-character loop and never rolls back past the first token. + const int64_t end_index = std::max(1, static_cast(ids.size()) - stream_config_.unfixed_token_num); + return truncate_at_pipe(sanitize_utf8_lossy(tokenizer_.decode(std::vector(ids.begin(), ids.begin() + static_cast(end_index))))); + } + int64_t k = stream_config_.unfixed_token_num; + if (stream_config_.rollback_punctuation && ends_with_rollback_punctuation(raw_truncated)) { + k = 0; + } + return truncate_at_pipe(decode_rollback_prefix(ids, k)); +} + +R2T2ASRSession::StreamOutcome R2T2ASRSession::decode_stream_chunk(bool final_flush) { + StreamOutcome outcome; + const std::string prefix = build_stream_prefix(final_flush); + + runtime::AudioBuffer accum; + accum.sample_rate = stream_sample_rate_ > 0 ? stream_sample_rate_ : assets_->config.sample_rate; + accum.channels = stream_channels_; + accum.samples = audio_accum_; + const auto features = frontend_.extract(accum); + const auto prompt = tokenizer_.build_raw_audio_prompt(prompt_raw_ + prefix, features.encoder_tokens); + const auto embeddings = audio_encoder_.encode(features); + std::string generated = generate_text(prompt, embeddings); + generated = normalize_punct_by_context(generated); + generated = sanitize_utf8_lossy(generated); + // Remove U+FFFD replacement characters, mirroring .replace('\ufffd', ''). + for (size_t pos = 0; (pos = generated.find(kReplacementChar, pos)) != std::string::npos;) { + generated.erase(pos, std::char_traits::length(kReplacementChar)); + } + + raw_decoded_ = prefix + generated; + + std::string detected; + if (force_language_.empty()) { + detected = parse_language_output(raw_decoded_, std::string()).language; + } + if (force_language_ == "Chinese" || detected == "Chinese") { + raw_decoded_ = remove_spaces_between_chinese(raw_decoded_); + } + const auto parsed = parse_asr_output(raw_decoded_, force_language_); + if (contains_asr_text_tag(raw_decoded_)) { + raw_decoded_ = text_before_asr_tag(raw_decoded_) + kAsrTextTag + parsed.text; + } else { + raw_decoded_ = parsed.text; + } + raw_decoded_ = truncate_at_pipe(raw_decoded_); + + const auto current_ids = tokenizer_.encode(raw_decoded_); + int64_t k = stream_config_.unfixed_token_num; + if (stream_config_.rollback_punctuation && ends_with_rollback_punctuation(raw_decoded_)) { + k = 0; + } + if (contains_asr_text_tag(raw_decoded_) && text_after_asr_tag(raw_decoded_).empty()) { + k = 0; + } + std::string fixed_text = decode_rollback_prefix(current_ids, k); + if (contains_asr_text_tag(fixed_text)) { + fixed_text = text_after_asr_tag(fixed_text); + } + fixed_text = truncate_at_pipe(fixed_text); + + if (!contains_asr_text_tag(raw_decoded_) && force_language_.empty()) { + // The model has not emitted the language tag yet: nothing to commit. + text_.clear(); + debug::trace_log_scalar("r2t2_asr.stream.final_flush", final_flush ? 1 : 0); + debug::trace_log_scalar("r2t2_asr.stream.chunk_id", chunk_id_); + debug::trace_log_scalar("r2t2_asr.stream.raw_decoded", raw_decoded_); + debug::trace_log_scalar("r2t2_asr.stream.fixed_text", std::string_view{}); + debug::trace_log_scalar("r2t2_asr.stream.text", std::string_view{}); + return outcome; + } + + language_ = parsed.language; + text_ = truncate_at_pipe(parsed.text); + ++chunk_id_; + outcome.text = text_; + outcome.fixed_text = fixed_text; + debug::trace_log_scalar("r2t2_asr.stream.final_flush", final_flush ? 1 : 0); + debug::trace_log_scalar("r2t2_asr.stream.chunk_id", chunk_id_); + debug::trace_log_scalar("r2t2_asr.stream.raw_decoded", raw_decoded_); + debug::trace_log_scalar("r2t2_asr.stream.fixed_text", fixed_text); + debug::trace_log_scalar("r2t2_asr.stream.text", text_); + return outcome; +} + +void R2T2ASRSession::publish_stream_delta(const std::string & fixed_text, runtime::StreamEvent & event) { + // Mirrors the reference WebSocket integrator, which slices the committed + // text by the previously published length (in code points): + // + // if len(fixed) > len(last_fixed): emit fixed[len(last_fixed):] + // + // The stable prefix can regress between chunks (for example a token + // rollback can leave a partial metadata fragment such as "language"), and + // the reference does not rewrite what it already sent. The authoritative + // transcript is delivered in the final result, so consumers that need exact + // text use that. + const size_t length = utf8_codepoint_count(fixed_text); + if (length <= published_codepoints_) { + return; + } + runtime::Transcript transcript; + transcript.text = utf8_slice_from_codepoint(fixed_text, published_codepoints_); + if (transcript.text.empty()) { + return; + } + transcript.language = language_; + event.partial_text = std::move(transcript); + published_codepoints_ = length; +} + +runtime::StreamingPolicy R2T2ASRSession::streaming_policy() const { + runtime::StreamingPolicy policy; + policy.input = runtime::StreamingInputKind::AudioChunks; + policy.output = runtime::StreamingOutputKind::FinalResult; + policy.preferred_audio_chunk_seconds = stream_config_.chunk_seconds; + return policy; +} + +void R2T2ASRSession::start_stream(const runtime::TaskRequest & request) { + require_prepared("R2T2 ASR start_stream()"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error("R2T2 ASR start_stream() requires a streaming session"); + } + reset(); + streaming_request_ = request; + if (streaming_request_.audio_input.has_value()) { + streaming_request_.audio_input->samples.clear(); + } + context_ = streaming_request_.text_input.has_value() ? streaming_request_.text_input->text : std::string(); + force_language_ = streaming_request_.text_input.has_value() ? streaming_request_.text_input->language : std::string(); + if (const auto value = runtime::find_option(streaming_request_.options, {"language"})) { + force_language_ = *value == "Auto" ? std::string() : *value; + } + if (!force_language_.empty()) { + force_language_ = resolve_language(force_language_); + if (!language_is_supported(*assets_, force_language_)) { + throw std::runtime_error("R2T2 ASR language is not supported by this model: " + force_language_); + } + } + prompt_raw_ = tokenizer_.build_prompt_text(context_, force_language_); + stream_started_ = true; + stream_wall_start_ = Clock::now(); +} + +void R2T2ASRSession::set_stream_event_sink(runtime::StreamEventCallback sink) { + stream_event_sink_ = std::move(sink); +} + +void R2T2ASRSession::reset() { + require_prepared("R2T2 ASR reset()"); + streaming_request_ = runtime::TaskRequest{}; + streaming_result_ = runtime::TaskResult{}; + prompt_raw_.clear(); + force_language_.clear(); + context_.clear(); + language_.clear(); + text_.clear(); + raw_decoded_.clear(); + buffer_.clear(); + audio_accum_.clear(); + chunk_size_samples_ = 0; + chunk_id_ = 0; + published_codepoints_ = 0; + stream_sample_rate_ = 0; + stream_channels_ = 1; + stream_started_ = false; + stream_wall_start_ = {}; +} + +runtime::StreamEvent R2T2ASRSession::process_audio_chunk(const runtime::AudioChunk & chunk) { + require_prepared("R2T2 ASR process_audio_chunk()"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error("R2T2 ASR process_audio_chunk() requires a streaming session"); + } + if (!stream_started_) { + throw std::runtime_error("R2T2 ASR process_audio_chunk() requires start_stream"); + } + if (chunk.sample_rate <= 0 || chunk.channels <= 0 || + chunk.samples.size() % static_cast(chunk.channels) != 0) { + throw std::runtime_error("R2T2 ASR streaming audio chunk has invalid layout"); + } + if (chunk_size_samples_ == 0) { + stream_sample_rate_ = chunk.sample_rate; + stream_channels_ = chunk.channels; + chunk_size_samples_ = std::max( + 1, + static_cast(std::llround(stream_config_.chunk_seconds * static_cast(chunk.sample_rate)))); + } else if (chunk.sample_rate != stream_sample_rate_ || chunk.channels != stream_channels_) { + // Chunk boundaries are counted in frames of the stream's first chunk; + // a mid-stream format change would silently corrupt the slicing. + throw std::runtime_error( + "R2T2 ASR streaming audio format changed mid-stream (sample rate or channel count); start a new stream instead"); + } + buffer_.insert(buffer_.end(), chunk.samples.begin(), chunk.samples.end()); + + runtime::StreamEvent event; + event.is_final = false; + const size_t channel_stride = static_cast(chunk.channels); + while (buffer_.size() >= static_cast(chunk_size_samples_) * channel_stride) { + const size_t take_values = static_cast(chunk_size_samples_) * channel_stride; + audio_accum_.insert(audio_accum_.end(), buffer_.begin(), buffer_.begin() + static_cast(take_values)); + buffer_.erase(buffer_.begin(), buffer_.begin() + static_cast(take_values)); + const auto outcome = decode_stream_chunk(/*final_flush=*/false); + if (!outcome.fixed_text.empty()) { + publish_stream_delta(outcome.fixed_text, event); + } + if (!streaming_result_.text_output.has_value()) { + streaming_result_.text_output = runtime::Transcript{outcome.text, language_}; + } else { + streaming_result_.text_output->text = outcome.text; + if (!language_.empty()) { + streaming_result_.text_output->language = language_; + } + } + if (stream_event_sink_ != nullptr && event.partial_text.has_value()) { + stream_event_sink_(event); + event.partial_text.reset(); + } + } + if (stream_event_sink_ != nullptr && event.partial_text.has_value()) { + stream_event_sink_(event); + event.partial_text.reset(); + } + return event; +} + +runtime::TaskResult R2T2ASRSession::finish_stream() { + return finalize(); +} + +runtime::TaskResult R2T2ASRSession::finalize() { + const auto finalize_start = Clock::now(); + require_prepared("R2T2 ASR finalize()"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error("R2T2 ASR finalize() requires a streaming session"); + } + if (!stream_started_) { + throw std::runtime_error("R2T2 ASR finalize() requires start_stream"); + } + if (!buffer_.empty()) { + audio_accum_.insert(audio_accum_.end(), buffer_.begin(), buffer_.end()); + buffer_.clear(); + const auto outcome = decode_stream_chunk(/*final_flush=*/true); + if (!streaming_result_.text_output.has_value()) { + streaming_result_.text_output = runtime::Transcript{outcome.text, language_}; + } else { + streaming_result_.text_output->text = outcome.text; + if (!language_.empty()) { + streaming_result_.text_output->language = language_; + } + } + } + if (!streaming_result_.text_output.has_value()) { + streaming_result_.text_output = runtime::Transcript{text_, language_}; + } + if (stream_event_sink_ != nullptr) { + // The final transcript travels in the task result (the server emits it + // as transcript.text.done); the reference integrator adds no final + // delta here either. + runtime::StreamEvent event; + event.is_final = true; + stream_event_sink_(event); + } + stream_started_ = false; + debug::timing_log_scalar("r2t2_asr.session.stream.chunks", chunk_id_); + debug::timing_log_scalar("r2t2_asr.session.stream.finalize_ms", engine::debug::elapsed_ms(finalize_start)); + if (stream_wall_start_ != std::chrono::steady_clock::time_point{}) { + debug::timing_log_scalar("r2t2_asr.session.stream.wall_ms", engine::debug::elapsed_ms(stream_wall_start_)); + debug::timing_log_scalar("session.wall_ms", engine::debug::elapsed_ms(stream_wall_start_)); + } + return streaming_result_; +} + +// Loading adapter: r2t2_asr uses the schema-v1 spec-backed loader, so the loader +// wiring stays beside the session it constructs (no per-model loader.{h,cpp}). +std::shared_ptr make_r2t2_asr_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = "r2t2_asr"; + config.load_assets = [](const std::filesystem::path & model_path) { + return load_r2t2_asr_assets(model_path); + }; + config.create_session = [](const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + (void) contract; + return std::make_unique(task, options, std::move(assets)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::community_models::r2t2_asr diff --git a/src/community_models/r2t2_asr/text_postprocess.cpp b/src/community_models/r2t2_asr/text_postprocess.cpp new file mode 100644 index 000000000..38d583c40 --- /dev/null +++ b/src/community_models/r2t2_asr/text_postprocess.cpp @@ -0,0 +1,605 @@ +#include "engine/community_models/r2t2_asr/text_postprocess.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::r2t2_asr { +namespace { + +std::vector utf8_to_codepoints(const std::string & text) { + std::vector out; + out.reserve(text.size()); + size_t i = 0; + while (i < text.size()) { + const auto byte = static_cast(text[i]); + if (byte < 0x80) { + out.push_back(byte); + i += 1; + } else if ((byte >> 5) == 0x6 && i + 1 < text.size() && + (static_cast(text[i + 1]) >> 6) == 0x2) { + out.push_back(((byte & 0x1F) << 6) | (static_cast(text[i + 1]) & 0x3F)); + i += 2; + } else if ((byte >> 4) == 0xE && i + 2 < text.size() && + (static_cast(text[i + 1]) >> 6) == 0x2 && + (static_cast(text[i + 2]) >> 6) == 0x2) { + out.push_back(((byte & 0x0F) << 12) | ((static_cast(text[i + 1]) & 0x3F) << 6) | + (static_cast(text[i + 2]) & 0x3F)); + i += 3; + } else if ((byte >> 3) == 0x1E && i + 3 < text.size() && + (static_cast(text[i + 1]) >> 6) == 0x2 && + (static_cast(text[i + 2]) >> 6) == 0x2 && + (static_cast(text[i + 3]) >> 6) == 0x2) { + out.push_back(((byte & 0x07) << 18) | ((static_cast(text[i + 1]) & 0x3F) << 12) | + ((static_cast(text[i + 2]) & 0x3F) << 6) | + (static_cast(text[i + 3]) & 0x3F)); + i += 4; + } else { + out.push_back(0xFFFD); + i += 1; + } + } + return out; +} + +void append_utf8(std::string & out, uint32_t codepoint) { + if (codepoint < 0x80) { + out.push_back(static_cast(codepoint)); + } else if (codepoint < 0x800) { + out.push_back(static_cast(0xC0 | (codepoint >> 6))); + out.push_back(static_cast(0x80 | (codepoint & 0x3F))); + } else if (codepoint < 0x10000) { + out.push_back(static_cast(0xE0 | (codepoint >> 12))); + out.push_back(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (codepoint & 0x3F))); + } else { + out.push_back(static_cast(0xF0 | (codepoint >> 18))); + out.push_back(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + out.push_back(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (codepoint & 0x3F))); + } +} + +std::string codepoints_to_utf8(const std::vector & codepoints) { + std::string out; + out.reserve(codepoints.size() * 3); + for (const uint32_t codepoint : codepoints) { + append_utf8(out, codepoint); + } + return out; +} + +bool is_chinese_codepoint(uint32_t codepoint) { + return codepoint >= 0x4E00 && codepoint <= 0x9FFF; +} + +bool is_ascii_digit_or_letter(uint32_t codepoint) { + return (codepoint >= '0' && codepoint <= '9') || (codepoint >= 'a' && codepoint <= 'z') || + (codepoint >= 'A' && codepoint <= 'Z'); +} + +bool is_python_space(uint32_t codepoint) { + switch (codepoint) { + case 0x20: // space + case 0x09: // \t + case 0x0A: // \n + case 0x0B: + case 0x0C: + case 0x0D: // \r + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x85: + case 0xA0: + case 0x1680: + case 0x2000: + case 0x2001: + case 0x2002: + case 0x2003: + case 0x2004: + case 0x2005: + case 0x2006: + case 0x2007: + case 0x2008: + case 0x2009: + case 0x200A: + case 0x2028: + case 0x2029: + case 0x202F: + case 0x205F: + case 0x3000: + return true; + default: + return false; + } +} + +/// The punctuation set matched by _ALL_PUNCT_PAT in the reference code. +bool is_normalized_punctuation(uint32_t codepoint) { + switch (codepoint) { + case ',': + case '.': + case '!': + case '?': + case ';': + case ':': + case '(': + case ')': + case 0xFF0C: // ， + case 0x3002: // 。 + case 0xFF01: // ！ + case 0xFF1F: // ？ + case 0xFF1B: // ； + case 0xFF1A: // ： + case 0xFF08: // （ + case 0xFF09: // ） + return true; + default: + return false; + } +} + +uint32_t en_to_zh_punctuation(uint32_t codepoint) { + switch (codepoint) { + case ',': + return 0xFF0C; + case '.': + return 0x3002; + case '!': + return 0xFF01; + case '?': + return 0xFF1F; + case ';': + return 0xFF1B; + case ':': + return 0xFF1A; + case '(': + return 0xFF08; + case ')': + return 0xFF09; + default: + return codepoint; + } +} + +uint32_t zh_to_en_punctuation(uint32_t codepoint) { + switch (codepoint) { + case 0xFF0C: + return ','; + case 0x3002: + return '.'; + case 0xFF01: + return '!'; + case 0xFF1F: + return '?'; + case 0xFF1B: + return ';'; + case 0xFF1A: + return ':'; + case 0xFF08: + return '('; + case 0xFF09: + return ')'; + default: + return codepoint; + } +} + +bool is_ascii_alnum_or_quote(uint32_t codepoint) { + return is_ascii_digit_or_letter(codepoint) || codepoint == '"' || codepoint == '\''; +} + +std::vector fix_char_repeats(const std::vector & s, int threshold) { + std::vector out; + size_t i = 0; + const size_t n = s.size(); + while (i < n) { + size_t count = 1; + while (i + count < n && s[i + count] == s[i]) { + ++count; + } + if (static_cast(count) > threshold) { + out.push_back(s[i]); + } else { + out.insert(out.end(), s.begin() + static_cast(i), s.begin() + static_cast(i + count)); + } + i += count; + } + return out; +} + +std::vector fix_pattern_repeats(const std::vector & s, int threshold, int max_len) { + const size_t n = s.size(); + const size_t min_repeat_chars = static_cast(threshold) * 2; + if (n < min_repeat_chars) { + return s; + } + std::vector result; + size_t i = 0; + bool exhausted = false; + while (i + min_repeat_chars <= n) { + bool found = false; + for (int k = 1; k <= max_len; ++k) { + const size_t pattern_length = static_cast(k); + if (i + static_cast(k) * static_cast(threshold) > n) { + break; + } + const auto pattern_begin = s.begin() + static_cast(i); + const auto pattern = std::vector(pattern_begin, pattern_begin + static_cast(pattern_length)); + bool valid = true; + for (int rep = 1; rep < threshold; ++rep) { + const size_t start_idx = i + static_cast(rep) * pattern_length; + if (std::vector(s.begin() + static_cast(start_idx), s.begin() + static_cast(start_idx + pattern_length)) != pattern) { + valid = false; + break; + } + } + if (valid) { + size_t end_index = i + pattern_length * static_cast(threshold); + while (end_index + pattern_length <= n && + std::vector(s.begin() + static_cast(end_index), s.begin() + static_cast(end_index + pattern_length)) == pattern) { + end_index += pattern_length; + } + result.insert(result.end(), pattern.begin(), pattern.end()); + const auto rest = fix_pattern_repeats( + std::vector(s.begin() + static_cast(end_index), s.end()), threshold, max_len); + result.insert(result.end(), rest.begin(), rest.end()); + i = n; + found = true; + break; + } + } + if (found) { + exhausted = true; + break; + } + result.push_back(s[i]); + ++i; + } + if (!exhausted && i <= n) { + result.insert(result.end(), s.begin() + static_cast(std::min(i, n)), s.end()); + } + return result; +} + +/// str.strip() semantics over the Python whitespace set. +std::string strip_python(const std::string & text) { + const auto codepoints = utf8_to_codepoints(text); + size_t begin = 0; + size_t end = codepoints.size(); + while (begin < end && is_python_space(codepoints[begin])) { + ++begin; + } + while (end > begin && is_python_space(codepoints[end - 1])) { + --end; + } + return codepoints_to_utf8(std::vector(codepoints.begin() + static_cast(begin), codepoints.begin() + static_cast(end))); +} + +bool contains_ascii_needle(const std::string & text, const char * needle) { + return text.find(needle) != std::string::npos; +} + +bool starts_with_ascii(const std::string & text, const char * prefix) { + const size_t length = std::char_traits::length(prefix); + if (text.size() < length) { + return false; + } + return text.compare(0, length, prefix) == 0; +} + +std::string to_lower_ascii(const std::string & text) { + std::string out = text; + for (char & ch : out) { + if (ch >= 'A' && ch <= 'Z') { + ch += 'a' - 'A'; + } + } + return out; +} + +R2T2ParsedOutput parse_tagged_output(const std::string & raw, const std::string & user_language, bool apply_repetition_fix) { + if (user_language.empty()) { + std::string s = strip_python(raw); + if (s.empty()) { + return {}; + } + if (apply_repetition_fix) { + s = detect_and_fix_repetitions(s); + } + if (!contains_asr_text_tag(s)) { + return {std::string(), strip_python(s)}; + } + const std::string meta_part = text_before_asr_tag(s); + const std::string text_part = text_after_asr_tag(s); + const std::string meta_lower = to_lower_ascii(meta_part); + if (meta_lower.find("language none") != std::string::npos) { + const std::string t = strip_python(text_part); + if (t.empty()) { + return {}; + } + return {std::string(), t}; + } + std::string language; + size_t line_begin = 0; + while (line_begin <= meta_part.size()) { + const size_t line_end = meta_part.find('\n', line_begin); + const std::string line = strip_python(meta_part.substr( + line_begin, + line_end == std::string::npos ? std::string::npos : line_end - line_begin)); + if (!line.empty()) { + if (starts_with_ascii(line, kLanguagePrefix)) { + const std::string value = strip_python(line.substr(std::char_traits::length(kLanguagePrefix))); + if (!value.empty()) { + language = normalize_language_name(value); + } + break; + } + } + if (line_end == std::string::npos) { + break; + } + line_begin = line_end + 1; + } + return {language, strip_python(text_part)}; + } + // Forced language: the model output is treated as pure transcription text. + std::string s = strip_python(raw); + if (!s.empty() && apply_repetition_fix) { + s = detect_and_fix_repetitions(s); + } + return {user_language, s}; +} + +} // namespace + +std::string sanitize_utf8_lossy(const std::string & text) { + std::string out; + out.reserve(text.size() + 16); + size_t i = 0; + while (i < text.size()) { + const auto byte = static_cast(text[i]); + const size_t remaining = text.size() - i; + if (byte < 0x80) { + out.push_back(text[i]); + i += 1; + continue; + } + size_t expected = 0; + if ((byte >> 5) == 0x6) { + expected = 2; + } else if ((byte >> 4) == 0xE) { + expected = 3; + } else if ((byte >> 3) == 0x1E) { + expected = 4; + } + bool valid = false; + if (expected != 0 && remaining >= expected) { + valid = true; + for (size_t j = 1; j < expected; ++j) { + if ((static_cast(text[i + j]) >> 6) != 0x2) { + valid = false; + break; + } + } + } + if (valid) { + out.append(text, i, expected); + i += expected; + } else { + out.append(kReplacementChar); + i += 1; + } + } + return out; +} + +std::string normalize_punct_by_context(const std::string & text) { + const auto codepoints = utf8_to_codepoints(text); + std::vector out; + out.reserve(codepoints.size()); + for (size_t pos = 0; pos < codepoints.size(); ++pos) { + const uint32_t punct = codepoints[pos]; + if (!is_normalized_punctuation(punct)) { + out.push_back(punct); + continue; + } + // Find the closest preceding non-space character. + uint32_t previous = 0; + bool has_previous = false; + for (size_t back = pos; back-- > 0;) { + if (!is_python_space(codepoints[back])) { + previous = codepoints[back]; + has_previous = true; + break; + } + } + if (!has_previous) { + out.push_back(punct); + continue; + } + if (is_chinese_codepoint(previous)) { + out.push_back(en_to_zh_punctuation(punct)); + } else if (previous < 0x80 && is_ascii_alnum_or_quote(previous)) { + out.push_back(zh_to_en_punctuation(punct)); + } else { + out.push_back(punct); + } + } + return codepoints_to_utf8(out); +} + +std::string detect_and_fix_repetitions(const std::string & text, int threshold) { + auto codepoints = utf8_to_codepoints(text); + codepoints = fix_char_repeats(codepoints, threshold); + codepoints = fix_pattern_repeats(codepoints, threshold, 20); + return codepoints_to_utf8(codepoints); +} + +std::string remove_spaces_between_chinese(const std::string & text) { + const auto codepoints = utf8_to_codepoints(text); + std::vector out; + out.reserve(codepoints.size()); + size_t i = 0; + while (i < codepoints.size()) { + if (is_python_space(codepoints[i])) { + size_t j = i; + while (j < codepoints.size() && is_python_space(codepoints[j])) { + ++j; + } + const bool between_chinese = i > 0 && j < codepoints.size() && + is_chinese_codepoint(codepoints[i - 1]) && is_chinese_codepoint(codepoints[j]); + if (!between_chinese) { + out.insert(out.end(), codepoints.begin() + static_cast(i), codepoints.begin() + static_cast(j)); + } + i = j; + continue; + } + out.push_back(codepoints[i]); + ++i; + } + return codepoints_to_utf8(out); +} + +std::string normalize_language_name(const std::string & language) { + const auto codepoints = utf8_to_codepoints(language); + size_t begin = 0; + size_t end = codepoints.size(); + while (begin < end && is_python_space(codepoints[begin])) { + ++begin; + } + while (end > begin && is_python_space(codepoints[end - 1])) { + --end; + } + if (begin >= end) { + throw std::runtime_error("language is empty"); + } + std::string out; + for (size_t i = begin; i < end; ++i) { + uint32_t codepoint = codepoints[i]; + if (i == begin) { + if (codepoint >= 'a' && codepoint <= 'z') { + codepoint -= 'a' - 'A'; + } + } else if (codepoint >= 'A' && codepoint <= 'Z') { + codepoint += 'a' - 'A'; + } + append_utf8(out, codepoint); + } + return out; +} + +std::string resolve_language(const std::string & language) { + const std::string trimmed = strip_python(language); + if (trimmed.empty()) { + return {}; + } + // The model spec and the WebUI speak ISO-639 style codes; the R2T2 prompt + // wants the canonical names from config.json. + static constexpr std::pair kIsoToName[] = { + {"zh", "Chinese"}, {"en", "English"}, {"yue", "Cantonese"}, + {"ar", "Arabic"}, {"de", "German"}, {"fr", "French"}, + {"es", "Spanish"}, {"pt", "Portuguese"}, {"id", "Indonesian"}, + {"it", "Italian"}, {"ko", "Korean"}, {"ru", "Russian"}, + {"th", "Thai"}, {"vi", "Vietnamese"}, {"ja", "Japanese"}, + {"tr", "Turkish"}, {"hi", "Hindi"}, {"ms", "Malay"}, + {"nl", "Dutch"}, {"sv", "Swedish"}, {"da", "Danish"}, + {"fi", "Finnish"}, {"pl", "Polish"}, {"cs", "Czech"}, + {"fil", "Filipino"}, {"fa", "Persian"}, {"el", "Greek"}, + {"hu", "Hungarian"}, {"mk", "Macedonian"}, {"ro", "Romanian"}, + {"auto", ""}, {"none", ""}, + }; + const std::string lowered = to_lower_ascii(trimmed); + for (const auto & [code, name] : kIsoToName) { + if (lowered == code) { + return name; + } + } + return normalize_language_name(trimmed); +} + +R2T2ParsedOutput parse_language_output(const std::string & raw, const std::string & user_language) { // The reference strips only the right side when the forced language is + // exactly "English", and both sides otherwise. + std::string s; + if (user_language == "English") { + auto codepoints = utf8_to_codepoints(raw); + size_t end = codepoints.size(); + while (end > 0 && is_python_space(codepoints[end - 1])) { + --end; + } + s = codepoints_to_utf8(std::vector(codepoints.begin(), codepoints.begin() + static_cast(end))); + } else { + s = strip_python(raw); + } + if (s.empty()) { + return {}; + } + if (!user_language.empty()) { + return {user_language, s}; + } + return parse_tagged_output(s, std::string(), /*apply_repetition_fix=*/false); +} + +R2T2ParsedOutput parse_asr_output(const std::string & raw, const std::string & user_language) { + return parse_tagged_output(raw, user_language, /*apply_repetition_fix=*/true); +} + +std::string truncate_at_pipe(const std::string & text) { + const size_t pipe = text.find('|'); + return pipe == std::string::npos ? text : text.substr(0, pipe); +} + +bool contains_asr_text_tag(const std::string & text) { + return contains_ascii_needle(text, kAsrTextTag); +} + +std::string text_before_asr_tag(const std::string & text) { + const size_t tag = text.find(kAsrTextTag); + return tag == std::string::npos ? std::string() : text.substr(0, tag); +} + +std::string text_after_asr_tag(const std::string & text) { + const size_t tag = text.find(kAsrTextTag); + if (tag == std::string::npos) { + return {}; + } + return text.substr(tag + std::char_traits::length(kAsrTextTag)); +} + +std::size_t utf8_codepoint_count(const std::string & text) { + return utf8_to_codepoints(text).size(); +} + +std::string utf8_slice_from_codepoint(const std::string & text, std::size_t start_codepoint) { + const auto codepoints = utf8_to_codepoints(text); + if (start_codepoint >= codepoints.size()) { + return {}; + } + return codepoints_to_utf8(std::vector( + codepoints.begin() + static_cast(start_codepoint), + codepoints.end())); +} + +bool ends_with_rollback_punctuation(const std::string & trimmed_text) { + static constexpr std::array kPunct = { + 0xFF0C, 0x3002, 0xFF01, 0xFF1F, 0x3001, 0xFF1B, 0xFF1A, ',', '.', '!', '?', + }; + // The reference list also covers ':' and ';'. + const auto codepoints = utf8_to_codepoints(trimmed_text); + if (codepoints.empty()) { + return false; + } + const uint32_t last = codepoints.back(); + for (const uint32_t punct : kPunct) { + if (last == punct) { + return true; + } + } + return last == ':' || last == ';'; +} + +} // namespace engine::community_models::r2t2_asr diff --git a/src/community_models/r2t2_asr/thinker.cpp b/src/community_models/r2t2_asr/thinker.cpp new file mode 100644 index 000000000..8ab1acecb --- /dev/null +++ b/src/community_models/r2t2_asr/thinker.cpp @@ -0,0 +1,119 @@ +#include "engine/community_models/r2t2_asr/thinker.h" + +#include "engine/framework/runtime/greedy_qwen_decoder.h" + +#include +#include + +namespace engine::community_models::r2t2_asr { +namespace { + +namespace modules = engine::modules; + +runtime::GreedyQwenDecoderSpec make_decoder_spec(const R2T2ASRConfig & config) { + const auto & text = config.text_decoder; + runtime::GreedyQwenDecoderSpec spec; + // Qwen3-style stack: Q/K norms, no attention biases, separate QKV. + spec.decoder.stack.hidden_size = text.hidden_size; + spec.decoder.stack.num_attention_heads = text.num_attention_heads; + spec.decoder.stack.num_key_value_heads = text.num_key_value_heads; + spec.decoder.stack.head_dim = text.head_dim; + spec.decoder.stack.intermediate_size = text.intermediate_size; + spec.decoder.stack.layers = text.num_hidden_layers; + spec.decoder.stack.rms_norm_eps = text.rms_norm_eps; + spec.decoder.stack.rope_theta = text.rope_theta; + spec.decoder.stack.use_qk_norm = true; + spec.decoder.stack.runtime.static_cache.update_mode = + modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + spec.decoder.logits_size = text.output_size; + spec.decoder.logits_mode = modules::QwenCausalDecoderLogitsMode::LastStep; + spec.vocab_size = text.vocab_size; + spec.max_position_embeddings = text.max_position_embeddings; + spec.tie_word_embeddings = config.tie_word_embeddings; + spec.attention_bias = text.attention_bias; + spec.packed_qkv = false; + + // R2T2 ships the legacy `thinker.*` namespace and the tied LM head of the + // base Qwen3-ASR checkpoint; the HF layout is accepted for converted GGUFs. + const std::string model_prefix = config.hf_transformers_layout + ? "model.language_model" + : "thinker.model"; + spec.token_embedding_tensor = model_prefix + ".embed_tokens.weight"; + spec.final_norm_tensor = model_prefix + ".norm.weight"; + spec.layer_prefix = model_prefix + ".layers"; + spec.lm_head_tensor = config.hf_transformers_layout ? "lm_head.weight" : "thinker.lm_head.weight"; + spec.eos_token_ids = text.eos_token_ids; + return spec; +} + +} // namespace + +struct R2T2ASRThinkerRuntime::Impl { + Impl( + std::shared_ptr assets, + core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + assets::TensorStorageType weight_storage_type) + : config(assets == nullptr ? throw std::runtime_error("R2T2 ASR thinker requires assets") : assets->config), + runtime( + assets->model_weights, + make_decoder_spec(assets->config), + execution, + prefill_graph_arena_bytes, + decode_graph_arena_bytes, + weight_context_bytes, + weight_storage_type) {} + + R2T2ASRConfig config; + runtime::GreedyQwenDecoderRuntime runtime; +}; + +R2T2ASRThinkerRuntime::R2T2ASRThinkerRuntime( + std::shared_ptr assets, + core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique( + std::move(assets), + execution, + prefill_graph_arena_bytes, + decode_graph_arena_bytes, + weight_context_bytes, + weight_storage_type)) {} + +R2T2ASRThinkerRuntime::~R2T2ASRThinkerRuntime() = default; + +R2T2ASRGeneratedTokens R2T2ASRThinkerRuntime::generate( + const R2T2ASRPrompt & prompt, + const R2T2ASRAudioEmbeddings & audio_embeddings, + const R2T2ASRGenerationOptions & options) { + if (prompt.input_ids.empty()) { + throw std::runtime_error("R2T2 ASR thinker prompt is empty"); + } + const auto & text = impl_->config.text_decoder; + if (audio_embeddings.hidden_size != text.hidden_size || + audio_embeddings.tokens != static_cast(prompt.audio_token_positions.size()) || + static_cast(audio_embeddings.values.size()) != audio_embeddings.tokens * text.hidden_size) { + throw std::runtime_error("R2T2 ASR audio embeddings do not match the prompt placeholders"); + } + for (const int32_t position : prompt.audio_token_positions) { + if (position < 0 || position >= static_cast(prompt.input_ids.size())) { + throw std::runtime_error("R2T2 ASR audio placeholder position out of range"); + } + } + runtime::GreedyQwenDecoderRuntime::Prompt decoder_prompt; + decoder_prompt.input_ids = prompt.input_ids; + decoder_prompt.injection.values = audio_embeddings.values; + decoder_prompt.injection.tokens = audio_embeddings.tokens; + decoder_prompt.injection.positions = prompt.audio_token_positions; + + R2T2ASRGeneratedTokens out; + out.token_ids = impl_->runtime.generate(decoder_prompt, options.max_new_tokens); + return out; +} + +} // namespace engine::community_models::r2t2_asr diff --git a/src/community_models/r2t2_asr/tokenizer_text.cpp b/src/community_models/r2t2_asr/tokenizer_text.cpp new file mode 100644 index 000000000..bb2aa69dd --- /dev/null +++ b/src/community_models/r2t2_asr/tokenizer_text.cpp @@ -0,0 +1,115 @@ +#include "engine/community_models/r2t2_asr/tokenizer_text.h" + +#include "engine/framework/tokenizers/llama_bpe.h" + +#include +#include +#include + +namespace engine::community_models::r2t2_asr { + +struct R2T2ASRTextTokenizer::Impl { + std::shared_ptr tokenizer; +}; + +namespace { + +std::shared_ptr load_impl(const R2T2ASRAssets & assets) { + engine::tokenizers::LlamaBpeTokenizerSpec spec; + spec.tokenizer_config_path = assets.resources.require_file("tokenizer_config"); + if (const auto * path = assets.resources.find_file("vocab")) { + spec.vocab_path = *path; + } + if (const auto * path = assets.resources.find_file("merges")) { + spec.merges_path = *path; + } + if (const auto * path = assets.resources.find_file("tokenizer_json")) { + spec.tokenizer_json_path = *path; + } + spec.pre_type = engine::tokenizers::LlamaBpePreTokenizer::Qwen2; + + auto impl = std::make_shared(); + impl->tokenizer = engine::tokenizers::load_llama_bpe_tokenizer(spec); + return impl; +} + +std::string default_chat_prompt(const std::string & context) { + return "<|im_start|>system\n" + context + + "<|im_end|>\n<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|><|im_end|>\n<|im_start|>assistant\n"; +} + +} // namespace + +R2T2ASRTextTokenizer::R2T2ASRTextTokenizer(std::shared_ptr assets) + : assets_(std::move(assets)) { + if (assets_ == nullptr) { + throw std::runtime_error("R2T2 ASR tokenizer requires assets"); + } + impl_ = load_impl(*assets_); +} + +R2T2ASRPrompt R2T2ASRTextTokenizer::build_prompt( + const std::string & context, + const std::string & language, + const int64_t audio_feature_tokens) const { + return build_raw_audio_prompt(build_prompt_text(context, language), audio_feature_tokens); +} + +std::string R2T2ASRTextTokenizer::build_prompt_text( + const std::string & context, + const std::string & language) const { + std::string prompt = default_chat_prompt(context); + if (!language.empty() && language != "Auto") { + prompt += "language " + language + ""; + } + return prompt; +} + +std::vector R2T2ASRTextTokenizer::encode(const std::string & text) const { + return impl_->tokenizer->encode(text); +} + +R2T2ASRPrompt R2T2ASRTextTokenizer::build_raw_audio_prompt( + const std::string & text, + const int64_t audio_feature_tokens) const { + if (audio_feature_tokens <= 0) { + throw std::runtime_error("R2T2 ASR prompt requires positive audio feature token count"); + } + const auto ids = impl_->tokenizer->encode(text); + const int32_t audio_token = static_cast(assets_->config.text_decoder.audio_token_id); + std::vector expanded; + expanded.reserve(ids.size() + static_cast(audio_feature_tokens)); + R2T2ASRPrompt result; + for (const int32_t id : ids) { + if (id == audio_token) { + for (int64_t i = 0; i < audio_feature_tokens; ++i) { + result.audio_token_positions.push_back(static_cast(expanded.size())); + expanded.push_back(id); + } + } else { + expanded.push_back(id); + } + } + result.input_ids = std::move(expanded); + result.attention_mask.assign(result.input_ids.size(), 1); + return result; +} + +std::string R2T2ASRTextTokenizer::decode(const std::vector & token_ids) const { + std::vector filtered; + filtered.reserve(token_ids.size()); + for (const int32_t id : token_ids) { + if (id == assets_->config.text_decoder.pad_token_id || + std::find( + assets_->config.text_decoder.eos_token_ids.begin(), + assets_->config.text_decoder.eos_token_ids.end(), + static_cast(id)) != assets_->config.text_decoder.eos_token_ids.end() || + impl_->tokenizer->is_control_token_id(id)) { + continue; + } + filtered.push_back(id); + } + return impl_->tokenizer->decode(filtered); +} + +} // namespace engine::community_models::r2t2_asr diff --git a/tests/r2t2_asr/README.md b/tests/r2t2_asr/README.md new file mode 100644 index 000000000..27056e5e1 --- /dev/null +++ b/tests/r2t2_asr/README.md @@ -0,0 +1,63 @@ +# R2T2 ASR verification + +These files verify the `r2t2_asr` family against the macOS MPS reference +implementation in the Confucius4-R2T2 repository (see `docs/community_models/r2t2.md`). + +## Files + +| File | Purpose | +|---|---| +| `make_golden.py` | Runs the Python reference (`R2T2ASRModel` on MPS) for an audio file and writes offline text plus per-chunk streaming `fixed_text`, `raw_decoded`, and `text` to a golden JSON. | +| `compare.py` | Runs `audiocpp_cli` offline and streaming with `--log-file`, parses the per-chunk trace, and diffs everything against a golden. | +| `test_r2t2_asr_transcription.cpp` | Repo-native smoke test: offline + streaming transcripts against the golden for `assets/resources/sample_16k.wav`. Skips with exit code 125 when the model or audio is missing. Also exposes `--encode ` to dump token ids for tokenizer diffing. | +| `golden*.json` | Recorded reference outputs. | + +## Goldens + +| Golden | Audio | Reference run | +|---|---|---| +| `golden.json` | upstream `resources/test.wav` (Chinese) | MPS fp16, auto language, 320 ms chunks | +| `golden_zh.json` | same | MPS, forced `--language Chinese` | +| `golden_sample16k.json` | repo `assets/resources/sample_16k.wav` (English) | MPS fp16, auto language | +| `golden_sample16k_bf16.json` | same | MPS bf16 (isolates reference dtype) | + +Streaming parameters are fixed across goldens so comparisons are deterministic: +`chunk_size_ms=320`, `unfixed_chunk_num=2`, `unfixed_token_num=5`, +`max_new_tokens=32`, `rollback_punctuation=false`. + +## Regenerating a golden + +Run from the Confucius4-R2T2 checkout (its `uv` environment has torch/MPS): + +```bash +cd /path/to/Confucius4-R2T2 +PYTHONPATH=. uv run python /path/to/audio.cpp/tests/r2t2_asr/make_golden.py \ + --model_path /path/to/audio.cpp/models/Confucius4-R2T2 \ + --audio resources/test.wav \ + --out /path/to/audio.cpp/tests/r2t2_asr/golden.json +``` + +## Comparing + +```bash +cd /path/to/audio.cpp +python3 tests/r2t2_asr/compare.py \ + --cli build/macos-metal-release/bin/audiocpp_cli \ + --model models/Confucius4-R2T2 \ + --audio assets/resources/sample_16k.wav \ + --golden tests/r2t2_asr/golden_sample16k.json \ + --backend metal +``` + +The comparison checks four things: the offline transcript, the committed delta +stream (the reference WebSocket integrator's rule), every per-chunk committed +`fixed_text`, and the final streaming transcript. See the results table in +`docs/community_models/r2t2.md` for what is exact and the two documented internal +(non-observable) differences on the English clip. + +The same binary can dump the family tokenizer for diffing against Hugging Face: + +```bash +build/macos-metal-release/bin/test_r2t2_asr_transcription \ + --encode "language EnglishSome text 22,500" +``` diff --git a/tests/r2t2_asr/compare.py b/tests/r2t2_asr/compare.py new file mode 100644 index 000000000..1f4908508 --- /dev/null +++ b/tests/r2t2_asr/compare.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Compare audiocpp R2T2 output against the macOS MPS golden reference. + +Usage: + python3 tests/r2t2_asr/compare.py \ + --cli build/macos-metal-release/bin/audiocpp_cli \ + --model models/Confucius4-R2T2 \ + --audio \ + --golden tests/r2t2_asr/golden.json \ + [--backend metal] [--chunk-ms 320] + +Runs the offline CLI, then the streaming CLI with trace logging enabled, and +diffs both against golden.json produced by make_golden.py. +""" + +import argparse +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +TRACE_RE = re.compile(r"^\[TRACE [^\]]*\] (?P\S+)\s?(?P.*)$") + + +def run(cmd): + proc = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace") + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + sys.stderr.write(proc.stderr) + raise SystemExit(f"command failed ({proc.returncode}): {' '.join(map(str, cmd))}") + return proc + + +def parse_trace(path): + chunks = [] + pending = {} + for line in Path(path).read_text(encoding="utf-8").splitlines(): + m = TRACE_RE.match(line) + if not m: + continue + name = m.group("name") + value = m.group("value") + if name == "r2t2_asr.stream.chunk_id": + if pending: + chunks.append(pending) + pending = {"chunk_id": int(value), "final_flush": 0, "fixed_text": None, "text": None, "raw_decoded": None} + elif name == "r2t2_asr.stream.final_flush" and pending: + pending["final_flush"] = int(value) + elif name == "r2t2_asr.stream.fixed_text" and pending: + pending["fixed_text"] = value + elif name == "r2t2_asr.stream.raw_decoded" and pending: + pending["raw_decoded"] = value + elif name == "r2t2_asr.stream.text" and pending: + pending["text"] = value + if pending: + chunks.append(pending) + return chunks + + +def offline_text_from_golden(field): + """Extract the text from the Python repr of ASRTranscription(...).""" + m = re.search(r"""text=(['"])(.*)\1,\s*time_stamps=""", field, re.DOTALL) + if m: + return m.group(2) + return field + + +def transcript_from_stdout(stdout): + for line in stdout.splitlines(): + if line.startswith("text_output="): + return line[len("text_output="):] + for line in stdout.splitlines(): + if line.startswith("text="): + return line[len("text="):] + return None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cli", required=True) + ap.add_argument("--model", required=True) + ap.add_argument("--audio", required=True) + ap.add_argument("--golden", required=True) + ap.add_argument("--backend", default="metal") + ap.add_argument("--chunk-ms", type=int, default=None) + ap.add_argument("--max-new-tokens", type=int, default=None) + ap.add_argument("--offline-only", action="store_true") + args = ap.parse_args() + + golden = json.loads(Path(args.golden).read_text(encoding="utf-8")) + failures = [] + + # --- offline ----------------------------------------------------------- + offline_cmd = [ + args.cli, "--task", "asr", "--family", "r2t2_asr", "--model", args.model, + "--backend", args.backend, "--audio", args.audio, + ] + offline = run(offline_cmd) + got_offline = transcript_from_stdout(offline.stdout) + want_offline_text = offline_text_from_golden(golden["offline"]) + if got_offline != want_offline_text: + failures.append(("offline", want_offline_text, got_offline)) + print(f"offline MISMATCH\n want: {want_offline_text!r}\n got: {got_offline!r}") + else: + print(f"offline OK: {got_offline!r}") + + if args.offline_only: + return 1 if failures else 0 + + # --- streaming --------------------------------------------------------- + chunk_ms = args.chunk_ms if args.chunk_ms is not None else golden["chunk_ms"] + max_new_tokens = args.max_new_tokens if args.max_new_tokens is not None else golden["max_new_tokens"] + with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as tmp: + trace_path = tmp.name + stream_cmd = [ + args.cli, "--task", "asr", "--mode", "streaming", "--family", "r2t2_asr", + "--model", args.model, "--backend", args.backend, "--audio", args.audio, + "--session-option", f"r2t2_asr.chunk_size_ms={chunk_ms}", + "--session-option", f"r2t2_asr.max_tokens={max_new_tokens}", + "--session-option", f"r2t2_asr.unfixed_chunk_num={golden['unfixed_chunk_num']}", + "--session-option", f"r2t2_asr.unfixed_token_num={golden['unfixed_token_num']}", + "--log-file", trace_path, + ] + if golden.get("language"): + stream_cmd += ["--language", golden["language"]] + if golden.get("context"): + stream_cmd += ["--text", golden["context"]] + streams = run(stream_cmd) + got_chunks = parse_trace(trace_path) + got_final = transcript_from_stdout(streams.stdout) + + # Observable committed stream: the reference WebSocket integrator emits + # fixed_text[len(last):] whenever the stable prefix grows (code points). + last, expected_stream = "", [] + for chunk in golden["stream_chunks"]: + fixed = chunk["fixed_text"] + if len(fixed) > len(last): + expected_stream.append(fixed[len(last):]) + last = fixed + expected_stream_text = "".join(expected_stream) + got_stream_text = "".join( + line[len("partial_text="):] + for line in streams.stdout.splitlines() + if line.startswith("partial_text=") + ) + if got_stream_text != expected_stream_text: + failures.append(("committed_stream", expected_stream_text, got_stream_text)) + print(f"committed stream MISMATCH\n want: {expected_stream_text!r}\n got: {got_stream_text!r}") + else: + print(f"committed stream OK: {got_stream_text!r}") + + want_chunks = golden["stream_chunks"] + per_chunk = [c for c in got_chunks if not c["final_flush"]] + flush = [c for c in got_chunks if c["final_flush"]] + if len(per_chunk) != len(want_chunks): + failures.append(("chunk-count", len(want_chunks), len(per_chunk))) + print(f"chunk count differ: want {len(want_chunks)} got {len(per_chunk)}") + for i, (want, got) in enumerate(zip(want_chunks, per_chunk)): + if got["fixed_text"] != want["fixed_text"]: + failures.append((f"chunk[{i}].fixed_text", want["fixed_text"], got["fixed_text"])) + print(f"chunk[{i}] fixed_text MISMATCH\n want: {want['fixed_text']!r}\n got: {got['fixed_text']!r}") + print(f" raw_decoded: {got['raw_decoded']!r}") + if flush: + print(f"final flush text: {flush[-1]['text']!r}") + if got_final != golden["stream_final_text"]: + failures.append(("stream_final_text", golden["stream_final_text"], got_final)) + print(f"stream final MISMATCH\n want: {golden['stream_final_text']!r}\n got: {got_final!r}") + else: + print(f"stream final OK: {got_final!r}") + if not failures: + print(f"streaming OK: {len(per_chunk)} chunks match golden per chunk") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/r2t2_asr/golden.json b/tests/r2t2_asr/golden.json new file mode 100644 index 000000000..6bcd9c1b9 --- /dev/null +++ b/tests/r2t2_asr/golden.json @@ -0,0 +1,77 @@ +{ + "audio": "resources/test.wav", + "chunk_ms": 320, + "unfixed_chunk_num": 2, + "unfixed_token_num": 5, + "max_new_tokens": 32, + "language": null, + "context": "", + "offline": "[ASRTranscription(language='Chinese', text='之前有顾客自己带酒水，也没加收钱或者不让喝。', time_stamps=None)]", + "stream_chunks": [ + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "language" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "之前" + }, + { + "fixed_text": "之前有" + }, + { + "fixed_text": "之前有" + }, + { + "fixed_text": "之前有" + }, + { + "fixed_text": "之前有顾客" + }, + { + "fixed_text": "之前有顾客自己带" + }, + { + "fixed_text": "之前有顾客自己带酒" + }, + { + "fixed_text": "之前有顾客自己带酒" + }, + { + "fixed_text": "之前有顾客自己带酒水" + }, + { + "fixed_text": "之前有顾客自己带酒水也没" + }, + { + "fixed_text": "之前有顾客自己带酒水也没加" + }, + { + "fixed_text": "之前有顾客自己带酒水也没加" + }, + { + "fixed_text": "之前有顾客自己带酒水也没加" + } + ], + "stream_final_text": "之前有顾客自己带酒水也没加收钱或者不让喝", + "stream_finish_return": "之前有顾客自己带酒水也没加收钱或者不让喝" +} \ No newline at end of file diff --git a/tests/r2t2_asr/golden_sample16k.json b/tests/r2t2_asr/golden_sample16k.json new file mode 100644 index 000000000..0ba0b7b87 --- /dev/null +++ b/tests/r2t2_asr/golden_sample16k.json @@ -0,0 +1,229 @@ +{ + "audio": "/Users/david/github/voice/audio.cpp/assets/resources/sample_16k.wav", + "chunk_ms": 320, + "unfixed_chunk_num": 2, + "unfixed_token_num": 5, + "max_new_tokens": 32, + "language": null, + "context": "", + "offline": "[ASRTranscription(language='English', text=\"Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you.\", time_stamps=None)]", + "stream_chunks": [ + { + "fixed_text": "", + "raw_decoded": "language None", + "text": "" + }, + { + "fixed_text": "", + "raw_decoded": "language None", + "text": "" + }, + { + "fixed_text": "", + "raw_decoded": "language None", + "text": "" + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome", + "text": "Some" + }, + { + "fixed_text": "language", + "raw_decoded": "language EnglishSome call me", + "text": "Some call me" + }, + { + "fixed_text": "language", + "raw_decoded": "language EnglishSome call me", + "text": "Some call me" + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature.", + "text": "Some call me nature." + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature.", + "text": "Some call me nature." + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature.", + "text": "Some call me nature." + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature,", + "text": "Some call me nature," + }, + { + "fixed_text": "Some", + "raw_decoded": "language EnglishSome call me nature; others", + "text": "Some call me nature; others" + }, + { + "fixed_text": "Some", + "raw_decoded": "language EnglishSome call me nature; others", + "text": "Some call me nature; others" + }, + { + "fixed_text": "Some call", + "raw_decoded": "language EnglishSome call me nature; others call", + "text": "Some call me nature; others call" + }, + { + "fixed_text": "Some call me", + "raw_decoded": "language EnglishSome call me nature; others call me", + "text": "Some call me nature; others call me" + }, + { + "fixed_text": "Some call me nature", + "raw_decoded": "language EnglishSome call me nature, others call me mother", + "text": "Some call me nature, others call me mother" + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others call me", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've", + "text": "Some call me nature, others call me mother nature. I've" + }, + { + "fixed_text": "Some call me nature, others call me mother", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been", + "text": "Some call me nature, others call me mother nature. I've been" + }, + { + "fixed_text": "Some call me nature, others call me mother nature.", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for", + "text": "Some call me nature, others call me mother nature. I've been here for" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over", + "text": "Some call me nature, others call me mother nature. I've been here for over" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over", + "text": "Some call me nature, others call me mother nature. I've been here for over" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over", + "text": "Some call me nature, others call me mother nature. I've been here for over" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years. 22", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years. 22" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, ", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,00", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,00" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 2", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 2", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,5", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than" + } + ], + "stream_final_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you.", + "stream_finish_return": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you." +} \ No newline at end of file diff --git a/tests/r2t2_asr/golden_sample16k_bf16.json b/tests/r2t2_asr/golden_sample16k_bf16.json new file mode 100644 index 000000000..449e7fba3 --- /dev/null +++ b/tests/r2t2_asr/golden_sample16k_bf16.json @@ -0,0 +1,230 @@ +{ + "audio": "/Users/david/github/voice/audio.cpp/assets/resources/sample_16k.wav", + "chunk_ms": 320, + "unfixed_chunk_num": 2, + "unfixed_token_num": 5, + "max_new_tokens": 32, + "dtype": "bfloat16", + "language": null, + "context": "", + "offline": "[ASRTranscription(language='English', text=\"Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you.\", time_stamps=None)]", + "stream_chunks": [ + { + "fixed_text": "", + "raw_decoded": "language None", + "text": "" + }, + { + "fixed_text": "", + "raw_decoded": "language None", + "text": "" + }, + { + "fixed_text": "", + "raw_decoded": "language None", + "text": "" + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome", + "text": "Some" + }, + { + "fixed_text": "language", + "raw_decoded": "language EnglishSome call me", + "text": "Some call me" + }, + { + "fixed_text": "language", + "raw_decoded": "language EnglishSome call me", + "text": "Some call me" + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature.", + "text": "Some call me nature." + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature.", + "text": "Some call me nature." + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature.", + "text": "Some call me nature." + }, + { + "fixed_text": "", + "raw_decoded": "language EnglishSome call me nature,", + "text": "Some call me nature," + }, + { + "fixed_text": "Some", + "raw_decoded": "language EnglishSome call me nature; others", + "text": "Some call me nature; others" + }, + { + "fixed_text": "Some", + "raw_decoded": "language EnglishSome call me nature; others", + "text": "Some call me nature; others" + }, + { + "fixed_text": "Some call", + "raw_decoded": "language EnglishSome call me nature; others call", + "text": "Some call me nature; others call" + }, + { + "fixed_text": "Some call me", + "raw_decoded": "language EnglishSome call me nature, others call me", + "text": "Some call me nature, others call me" + }, + { + "fixed_text": "Some call me nature", + "raw_decoded": "language EnglishSome call me nature, others call me mother", + "text": "Some call me nature, others call me mother" + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature.", + "text": "Some call me nature, others call me mother nature." + }, + { + "fixed_text": "Some call me nature, others call me", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've", + "text": "Some call me nature, others call me mother nature. I've" + }, + { + "fixed_text": "Some call me nature, others call me mother", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been", + "text": "Some call me nature, others call me mother nature. I've been" + }, + { + "fixed_text": "Some call me nature, others call me mother nature.", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for", + "text": "Some call me nature, others call me mother nature. I've been here for" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over", + "text": "Some call me nature, others call me mother nature. I've been here for over" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over", + "text": "Some call me nature, others call me mother nature. I've been here for over" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over", + "text": "Some call me nature, others call me mother nature. I've been here for over" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years.", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years." + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years. 22", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years. 22" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 2", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,000", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,000" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 2", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 2", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer" + }, + { + "fixed_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,5", + "raw_decoded": "language EnglishSome call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than", + "text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than" + } + ], + "stream_final_text": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you.", + "stream_finish_return": "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you." +} \ No newline at end of file diff --git a/tests/r2t2_asr/golden_zh.json b/tests/r2t2_asr/golden_zh.json new file mode 100644 index 000000000..a282d8a89 --- /dev/null +++ b/tests/r2t2_asr/golden_zh.json @@ -0,0 +1,77 @@ +{ + "audio": "resources/test.wav", + "chunk_ms": 320, + "unfixed_chunk_num": 2, + "unfixed_token_num": 5, + "max_new_tokens": 32, + "language": "Chinese", + "context": "", + "offline": "[ASRTranscription(language='Chinese', text='之前有顾客自己带酒水，也没加收钱或者不让喝。', time_stamps=None)]", + "stream_chunks": [ + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "" + }, + { + "fixed_text": "之前" + }, + { + "fixed_text": "之前有" + }, + { + "fixed_text": "之前有" + }, + { + "fixed_text": "之前有" + }, + { + "fixed_text": "之前有顾客" + }, + { + "fixed_text": "之前有顾客自己带" + }, + { + "fixed_text": "之前有顾客自己带酒" + }, + { + "fixed_text": "之前有顾客自己带酒" + }, + { + "fixed_text": "之前有顾客自己带酒水" + }, + { + "fixed_text": "之前有顾客自己带酒水也没" + }, + { + "fixed_text": "之前有顾客自己带酒水也没加收" + }, + { + "fixed_text": "之前有顾客自己带酒水也没加收" + }, + { + "fixed_text": "之前有顾客自己带酒水也没加收" + } + ], + "stream_final_text": "之前有顾客自己带酒水也没加收钱或者不让喝。", + "stream_finish_return": "之前有顾客自己带酒水也没加收钱或者不让喝。" +} \ No newline at end of file diff --git a/tests/r2t2_asr/make_golden.py b/tests/r2t2_asr/make_golden.py new file mode 100644 index 000000000..aa1bb0f76 --- /dev/null +++ b/tests/r2t2_asr/make_golden.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Generate golden R2T2 outputs from the macOS MPS baseline. + +Run from the Confucius4-R2T2 repo (uv environment): + + cd /Users/david/github/voice/Confucius4-R2T2 + PYTHONPATH=. uv run python /Users/david/github/voice/audio.cpp/tests/r2t2_asr/make_golden.py \ + --model_path checkpoints/r2t2 --audio resources/test.wav \ + --out /Users/david/github/voice/audio.cpp/tests/r2t2_asr/golden.json + +Produces offline transcript plus the per-chunk (text, fixed_text) streaming +sequence used to verify the C++ LSP streaming port step by step. +""" + +import argparse +import json +import sys + +import numpy as np + +from r2t2 import R2T2ASRModel +from r2t2.r2t2_asr import ASRStreamingState # noqa: F401 (state shape doc) + + +def read_wav_16k(path): + import soundfile as sf + wav, sr = sf.read(path, dtype="float32", always_2d=False) + if wav.ndim > 1: + wav = wav.mean(axis=1) + if sr != 16000: + import librosa + wav = librosa.resample(wav, orig_sr=sr, target_sr=16000) + return np.asarray(wav, dtype=np.float32) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model_path", default="checkpoints/r2t2") + ap.add_argument("--audio", default="resources/test.wav") + ap.add_argument("--language", default=None) + ap.add_argument("--context", default="") + ap.add_argument("--chunk_ms", type=int, default=320) + ap.add_argument("--unfixed_chunk_num", type=int, default=2) + ap.add_argument("--unfixed_token_num", type=int, default=5) + ap.add_argument("--max_new_tokens", type=int, default=32) + ap.add_argument("--dtype", default=None, help="e.g. float16 or bfloat16 (default: MPS float16)") + ap.add_argument("--out", default="golden.json") + args = ap.parse_args() + + wav = read_wav_16k(args.audio) + asr = R2T2ASRModel.from_pretrained(args.model_path, dtype=args.dtype) + + golden = { + "audio": args.audio, + "chunk_ms": args.chunk_ms, + "unfixed_chunk_num": args.unfixed_chunk_num, + "unfixed_token_num": args.unfixed_token_num, + "max_new_tokens": args.max_new_tokens, + "dtype": args.dtype, + "language": args.language, + "context": args.context, + } + + offline = asr.transcribe((wav, 16000), context=args.context, language=args.language) + golden["offline"] = offline if isinstance(offline, str) else str(offline) + print("offline:", golden["offline"], file=sys.stderr) + + state = asr.init_streaming_state( + context=args.context, + language=args.language, + unfixed_chunk_num=args.unfixed_chunk_num, + unfixed_token_num=args.unfixed_token_num, + chunk_size_sec=args.chunk_ms / 1000.0, + ) + chunk_samples = int(round(args.chunk_ms / 1000.0 * 16000)) + chunks = [] + pos = 0 + while pos < wav.shape[0]: + seg = wav[pos:pos + chunk_samples] + pos += seg.shape[0] + before = state.chunk_id + _, fixed = asr.streaming_transcribe(seg, state, args.max_new_tokens) + if state.chunk_id == before: + # Buffered tail shorter than one chunk: no decode happened. + continue + chunks.append({ + "fixed_text": fixed, + "raw_decoded": state._raw_decoded, + "text": state.text, + }) + final = asr.finish_streaming_transcribe(state, args.max_new_tokens) + golden["stream_chunks"] = chunks + golden["stream_final_text"] = state.text + golden["stream_finish_return"] = final + print("stream final:", state.text, file=sys.stderr) + + with open(args.out, "w", encoding="utf-8") as f: + json.dump(golden, f, ensure_ascii=False, indent=1) + print("wrote", args.out, file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tests/r2t2_asr/test_r2t2_asr_transcription.cpp b/tests/r2t2_asr/test_r2t2_asr_transcription.cpp new file mode 100644 index 000000000..0273a0add --- /dev/null +++ b/tests/r2t2_asr/test_r2t2_asr_transcription.cpp @@ -0,0 +1,212 @@ +#include "engine/framework/audio/wav_reader.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/io/filesystem.h" +#include "engine/framework/runtime/model.h" +#include "engine/framework/runtime/registry.h" +#include "engine/framework/runtime/session.h" +#include "engine/community_models/r2t2_asr/assets.h" +#include "engine/community_models/r2t2_asr/tokenizer_text.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef ENGINE_REPO_ROOT +#define ENGINE_REPO_ROOT "." +#endif + +namespace { + +constexpr int kExitPass = 0; +constexpr int kExitFail = 1; +constexpr int kExitSkip = 125; + +// Golden output from the macOS MPS reference implementation +// (tests/r2t2_asr/golden_sample16k.json, produced by make_golden.py). +const char * kExpectedOffline = + "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you."; +const char * kExpectedStreamFinal = + "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you."; + +constexpr int64_t kStreamingChunkMs = 320; +constexpr int64_t kStreamingMaxNewTokens = 32; + +std::filesystem::path repo_path(const std::string & relative) { + return std::filesystem::path(ENGINE_REPO_ROOT) / relative; +} + +std::string arg_value(int argc, char ** argv, const std::string & name, const std::string & fallback) { + for (int i = 1; i + 1 < argc; ++i) { + if (argv[i] == name) { + return argv[i + 1]; + } + } + return fallback; +} + +engine::core::BackendType parse_backend(const std::string & value) { + if (value == "cpu") return engine::core::BackendType::Cpu; + if (value == "cuda") return engine::core::BackendType::Cuda; + if (value == "metal") return engine::core::BackendType::Metal; + if (value == "vulkan") return engine::core::BackendType::Vulkan; + if (value == "best") return engine::core::BackendType::BestAvailable; + throw std::runtime_error("unsupported backend: " + value); +} + +engine::runtime::AudioBuffer read_audio(const std::filesystem::path & path) { + const auto wav = engine::audio::read_wav_f32(path); + engine::runtime::AudioBuffer audio; + audio.sample_rate = wav.sample_rate; + audio.channels = wav.channels; + audio.samples = wav.samples; + return audio; +} + +std::string run_offline( + engine::runtime::ILoadedVoiceModel & model, + const engine::runtime::AudioBuffer & audio, + const engine::runtime::SessionOptions & options) { + auto session = model.create_task_session( + engine::runtime::TaskSpec{engine::runtime::VoiceTaskKind::Asr, engine::runtime::RunMode::Offline}, + options); + auto * offline = dynamic_cast(session.get()); + if (offline == nullptr) { + throw std::runtime_error("R2T2 session is not an offline session"); + } + offline->prepare(engine::runtime::build_preparation_request(audio)); + engine::runtime::TaskRequest request; + request.audio_input = audio; + const auto result = offline->run(request); + if (!result.text_output.has_value()) { + throw std::runtime_error("offline run produced no text"); + } + return result.text_output->text; +} + +std::string run_streaming( + engine::runtime::ILoadedVoiceModel & model, + const engine::runtime::AudioBuffer & audio, + const engine::runtime::SessionOptions & options) { + auto session = model.create_task_session( + engine::runtime::TaskSpec{engine::runtime::VoiceTaskKind::Asr, engine::runtime::RunMode::Streaming}, + options); + auto * streaming = dynamic_cast(session.get()); + if (streaming == nullptr) { + throw std::runtime_error("R2T2 session is not a streaming session"); + } + engine::runtime::TaskRequest request; + request.audio_input = audio; + streaming->prepare(engine::runtime::build_preparation_request(request)); + + std::string committed; + streaming->set_stream_event_sink([&](const engine::runtime::StreamEvent & event) { + if (event.partial_text.has_value()) { + committed += event.partial_text->text; + } + }); + streaming->start_stream(request); + + const int64_t chunk_frames = std::max( + 1, + static_cast(audio.sample_rate) * kStreamingChunkMs / 1000); + const int64_t frames = static_cast(audio.samples.size() / static_cast(audio.channels)); + for (int64_t start = 0; start < frames; start += chunk_frames) { + const int64_t take = std::min(chunk_frames, frames - start); + engine::runtime::AudioChunk chunk; + chunk.sample_rate = audio.sample_rate; + chunk.channels = audio.channels; + chunk.start_sample = start; + const auto begin = audio.samples.begin() + static_cast(start * audio.channels); + chunk.samples.assign(begin, begin + static_cast(take * audio.channels)); + streaming->process_audio_chunk(chunk); + } + const auto result = streaming->finish_stream(); + if (!result.text_output.has_value()) { + throw std::runtime_error("streaming run produced no text"); + } + std::cout << "Committed stream: " << committed << "\n"; + return result.text_output->text; +} + +// Debug aid: print token ids for a text argument so the family tokenizer can be +// diffed against the reference Hugging Face tokenizer. +int encode_probe(const std::filesystem::path & model_path, const std::string & text) { + auto assets = engine::community_models::r2t2_asr::load_r2t2_asr_assets(model_path, "r2t2_asr"); + engine::community_models::r2t2_asr::R2T2ASRTextTokenizer tokenizer(assets); + const auto ids = tokenizer.encode(text); + std::cout << "count=" << ids.size() << "\nids="; + for (size_t i = 0; i < ids.size(); ++i) { + std::cout << (i == 0 ? "" : ",") << ids[i]; + } + std::cout << "\n"; + return kExitPass; +} + +} // namespace + +int main(int argc, char ** argv) { + const std::filesystem::path model_path = arg_value(argc, argv, "--model", repo_path("models/Confucius4-R2T2").string()); + const auto encode_text = arg_value(argc, argv, "--encode", ""); + if (!encode_text.empty()) { + return encode_probe(model_path, encode_text); + } + const std::filesystem::path audio_path = arg_value(argc, argv, "--audio", repo_path("assets/resources/sample_16k.wav").string()); + const std::string backend_name = arg_value(argc, argv, "--backend", "best"); + + const bool model_available = + engine::io::is_existing_file(model_path) || + engine::io::is_existing_file(model_path / "config.json"); + if (!model_available || !engine::io::is_existing_file(audio_path)) { + std::fprintf( + stderr, + "SKIP: test_r2t2_asr_transcription requires model weights at '%s' and audio at '%s'.\n", + model_path.string().c_str(), + audio_path.string().c_str()); + return kExitSkip; + } + + try { + auto registry = engine::runtime::make_default_registry(); + engine::runtime::ModelLoadRequest load_request; + load_request.model_path = model_path; + load_request.family_hint = "r2t2_asr"; + auto model = registry.load(load_request); + + engine::runtime::SessionOptions options; + options.backend.type = parse_backend(backend_name); + options.backend.threads = 8; + options.options["r2t2_asr.chunk_size_ms"] = std::to_string(kStreamingChunkMs); + options.options["r2t2_asr.max_tokens"] = std::to_string(kStreamingMaxNewTokens); + + const auto audio = read_audio(audio_path); + + const std::string offline = run_offline(*model, audio, options); + std::cout << "Offline transcript: " << offline << "\n"; + if (offline != kExpectedOffline) { + std::cerr << "FAIL: offline transcript mismatch\n" + << " expected: " << kExpectedOffline << "\n" + << " actual: " << offline << "\n"; + return kExitFail; + } + + const std::string stream_final = run_streaming(*model, audio, options); + std::cout << "Streaming final: " << stream_final << "\n"; + if (stream_final != kExpectedStreamFinal) { + std::cerr << "FAIL: streaming transcript mismatch\n" + << " expected: " << kExpectedStreamFinal << "\n" + << " actual: " << stream_final << "\n"; + return kExitFail; + } + + std::cout << "PASS: Confucius4-R2T2 offline and streaming transcripts match the MPS golden.\n"; + return kExitPass; + } catch (const std::exception & error) { + std::cerr << "FAIL: " << error.what() << "\n"; + return kExitFail; + } +} diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json index f463efb11..ad387a4a5 100644 --- a/webui/configs/model_params.json +++ b/webui/configs/model_params.json @@ -111,6 +111,14 @@ {"name": "audio_chunk_duration_sec", "type": "number", "label": "audio_chunk_duration_sec", "default": 30, "minimum": 0.001, "step": 1} ], + "r2t2_asr": [ + {"name": "chunk_size_ms", "type": "slider", "scope": "session", "session_option": "r2t2_asr.chunk_size_ms", "label": "chunk_size_ms（流式分块毫秒）", "label_en": "chunk_size_ms (streaming chunk, ms)", "default": 320, "minimum": 80, "maximum": 2000, "step": 10, "precision": 0, "info": "80-2000ms：越小延迟越低；320ms 在 Apple Silicon 上延迟与速度较均衡。", "info_en": "80-2000 ms. Lower means lower latency; 320 ms balances latency and speed on Apple Silicon."}, + {"name": "unfixed_chunk_num", "type": "number", "scope": "session", "session_option": "r2t2_asr.unfixed_chunk_num", "label": "unfixed_chunk_num（前 N 块不用稳定前缀）", "label_en": "unfixed_chunk_num (leading chunks without prefix)", "default": 2, "minimum": 0, "maximum": 10, "step": 1, "precision": 0, "info": "开头若干块不使用已识别文本作为前缀提示。", "info_en": "Leading chunks that decode without a stable-prefix prompt."}, + {"name": "unfixed_token_num", "type": "number", "scope": "session", "session_option": "r2t2_asr.unfixed_token_num", "label": "unfixed_token_num（回滚 token 数）", "label_en": "unfixed_token_num (rollback tokens)", "default": 5, "minimum": 0, "maximum": 20, "step": 1, "precision": 0, "info": "作为前缀前从累积文本回滚的 token 数，用于降低边界抖动。", "info_en": "Tokens rolled back from the accumulated text before it is used as the prefix prompt."}, + {"name": "rollback_punctuation", "type": "bool", "scope": "session", "session_option": "r2t2_asr.rollback_punctuation", "label": "rollback_punctuation（句末标点不回滚）", "label_en": "rollback_punctuation (keep trailing punctuation)", "default": false, "info": "输出已以标点结尾时不再回滚 token。", "info_en": "Do not roll back tokens when the output already ends with punctuation."}, + {"name": "max_new_tokens", "type": "number", "scope": "session", "session_option": "r2t2_asr.max_tokens", "label": "max_new_tokens（每分块解码上限）", "label_en": "max_new_tokens (per-chunk decode budget)", "default": 32, "minimum": 1, "maximum": 256, "step": 1, "precision": 0, "info": "每个流式分块的贪婪解码上限（会话级，区别于离线 max_tokens）。", "info_en": "Greedy decode budget per streaming chunk (session-scoped; distinct from offline max_tokens)."} + ], + "pocket_tts": [ {"name": "frames_after_eos", "type": "number", "label": "frames_after_eos（-1=自动）", "default": -1, "minimum": -1, "step": 1, "precision": 0} ], diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 271811570..e176b5aaa 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -119,6 +119,8 @@ { "id": "qwen3-asr", "display_name": "Qwen3-ASR 0.6B (asr)", "family": "qwen3_asr", "path": "models/Qwen3-ASR-0.6B", "task": "asr", "mode": "offline", "download_id": "qwen3_asr_0_6b", "min_vram_gb": 3 }, { "id": "qwen3-asr-1.7b", "display_name": "Qwen3-ASR 1.7B HF (asr)", "family": "qwen3_asr", "path": "models/Qwen3-ASR-1.7B-hf", "task": "asr", "mode": "offline", "download_id": "qwen3_asr_1_7b_hf", "min_vram_gb": 6, "input_hint": "**Qwen3-ASR 1.7B**（HF 原生权重，免转换）：精度高于 0.6B；长音频自动分段转写；8G 卡显存偏紧，长音频建议先短段试跑。", "input_hint_en": "**Qwen3-ASR 1.7B**: native Hugging Face weights with no conversion required. It is more accurate than the 0.6B model and automatically chunks long audio; test short clips first on an 8 GB GPU." }, + { "id": "r2t2-asr", "display_name": "Confucius4-R2T2 (asr, 实时流式)", "display_name_en": "Confucius4-R2T2 (asr, real-time streaming)", "family": "r2t2_asr", "path": "models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", "task": "asr", "mode": "offline", "download_id": "r2t2_asr_q8_0", "min_vram_gb": 5, + "input_hint": "**Confucius4-R2T2**：网易有道实时流式 ASR，Qwen3-ASR-1.7B 微调，LSP 稳定前缀解码；提交文本永不回改，支持 80ms-2s 分块；Q8_0 GGUF（2.3G，自包含单文件），也支持 F16。", "input_hint_en": "**Confucius4-R2T2**: NetEase Youdao real-time streaming ASR, a Qwen3-ASR 1.7B fine-tune with Longest Stable Prefix decoding. Committed text is never revised; 80 ms-2 s chunks; Q8_0 GGUF (2.3 GB, self-contained single file), F16 also available." }, { "id": "niagara-asr-19m", "display_name": "Niagara ASR 19M (asr)", "display_name_en": "Niagara ASR 19M (asr)", "family": "niagara_asr", "path": "models/Niagara-ASR-GGUF/niagara-19m-batch.en-f32.gguf", "task": "asr", "mode": "offline", "download_id": "niagara_19m_f32", "min_vram_gb": 1, "input_hint": "**Niagara ASR 19M**：ABR 英语离线 ASR，F32 GGUF 权重。", "input_hint_en": "**Niagara ASR 19M**: ABR English offline ASR with F32 GGUF weights." }, { "id": "niagara-asr-38m", "display_name": "Niagara ASR 38M (asr)", "display_name_en": "Niagara ASR 38M (asr)", "family": "niagara_asr", "path": "models/Niagara-ASR-GGUF/niagara-38m-batch.en-f32.gguf", "task": "asr", "mode": "offline", "download_id": "niagara_38m_f32", "min_vram_gb": 1, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 23cc52eeb..e10d8f71f 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@
diff --git a/webui/native/src/lib/catalog.ts b/webui/native/src/lib/catalog.ts index efa88315f..2adf0c7a5 100644 --- a/webui/native/src/lib/catalog.ts +++ b/webui/native/src/lib/catalog.ts @@ -43,6 +43,7 @@ const exposeAllGgufPackageFamilies = new Set([ 'canary_asr', 'cohere_asr', 'moss_transcribe_diarize', + 'r2t2_asr', 'audiosr', 'controlfoley', 'breeze_tts', diff --git a/webui/native/src/routes/+page.svelte b/webui/native/src/routes/+page.svelte index 8ca2cb517..ce1b4acf3 100644 --- a/webui/native/src/routes/+page.svelte +++ b/webui/native/src/routes/+page.svelte @@ -315,7 +315,10 @@ }; const asrLanguages: Record = { canary_asr: ['en', 'de', 'es', 'fr'], - cohere_asr: ['en', 'fr', 'de', 'es', 'it', 'pt', 'nl', 'pl', 'el', 'ar', 'ja', 'zh', 'vi', 'ko'] + cohere_asr: ['en', 'fr', 'de', 'es', 'it', 'pt', 'nl', 'pl', 'el', 'ar', 'ja', 'zh', 'vi', 'ko'], + // Confucius4-R2T2 takes canonical language names (the engine normalizes + // case); 'Auto' leaves language detection on. + r2t2_asr: ['Auto', 'Chinese', 'English', 'Cantonese', 'Japanese', 'Korean', 'Arabic', 'German', 'French', 'Spanish', 'Portuguese', 'Indonesian', 'Italian', 'Russian', 'Thai', 'Vietnamese', 'Turkish', 'Hindi', 'Malay', 'Dutch', 'Swedish', 'Danish', 'Finnish', 'Polish', 'Czech', 'Filipino', 'Persian', 'Greek', 'Romanian', 'Hungarian', 'Macedonian'] }; function pathVariantLabel(path: string) { @@ -476,7 +479,7 @@ !['apollo', 'universr'].includes(selected?.family) && !replacesGenericControls.text; $: supportsLiveAsr = selected?.task === 'asr' && - ['voxtral_realtime', 'nemotron_asr', 'higgs_audio_stt', 'sense_asr', 'vibevoice_asr_streaming'].includes(selected?.family); + ['voxtral_realtime', 'nemotron_asr', 'higgs_audio_stt', 'sense_asr', 'vibevoice_asr_streaming', 'r2t2_asr'].includes(selected?.family); $: modelInventoryLoading = server === null || (Boolean(server.ui_management) && Object.keys(packageSizes).length === 0 && packageSizeState !== 'failed'); $: selectableModelIds = new Set(activeCatalog.filter((entry) => { @@ -1047,7 +1050,8 @@ !(hidesDurationSec && spec.name === 'duration_sec')); advancedValues = Object.fromEntries(byId.map((spec) => [spec.name, spec.default ?? ''])); if (selected?.family in asrTokenDefaults) asrMaxTokens = asrTokenDefaults[selected.family]; - if (selected?.family in asrLanguages) language = 'en'; + if (selected?.family === 'r2t2_asr') language = 'Auto'; + else if (selected?.family in asrLanguages) language = 'en'; if (selected?.family === 'minimax_h3') { duration = 15; advancedValues = { ...advancedValues, num_frames: miniMaxFramesForDuration(duration), dit_acceleration: 'none' }; From 3bab2867a7ecac055c62475bf1a72cb041928b11 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 11:28:20 +0900 Subject: [PATCH 2/9] fix(r2t2): exclude rollback metadata from streaming deltas --- docs/community_models/r2t2.md | 36 ++++++++----------- src/community_models/r2t2_asr/session.cpp | 17 ++++----- tests/r2t2_asr/compare.py | 17 +++++++-- .../r2t2_asr/test_r2t2_asr_transcription.cpp | 6 ++++ 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index 6a5712339..aeb62bdbf 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -151,23 +151,16 @@ then returns the complete transcript. ### Committed-text contract -Committed deltas follow the upstream WebSocket integrator exactly: +Committed deltas contain transcript text only. In automatic language mode, +a rolled-back prefix is not publishable until it contains ``; +metadata fragments such as `language` neither emit a delta nor advance the +published offset. Forced-language prompts already supply the metadata, so +untagged decoded prefixes remain valid transcript text in that mode. -```python -if len(fixed_text) > len(last_fixed_text): - emit fixed_text[len(last_fixed_text):] - last_fixed_text = fixed_text -``` - -Two consequences are inherited from the reference implementation and are -deliberate: - -* Lengths are counted in code points, so a delta is always valid UTF-8. -* The stable prefix can regress between chunks — a token rollback can leave a - partial metadata fragment such as `language` at the head of the committed - text, after which the next deltas slice past it. The authoritative transcript - is always delivered again in the final result (`transcript.text.done` on the - server), so consumers that need exact text should use that. +After metadata removal, growing stable prefixes emit only the suffix beyond +the previously published length, counted in Unicode code points. A shrinking +prefix emits nothing. The uncommitted tail is delivered in the final result +(`transcript.text.done` on the server), rather than as a final delta. The reference also ships a rolling-window variant for unbounded streams ("no reset": keep 16 s of audio, discard the oldest 8 s and the matching text). This @@ -318,9 +311,11 @@ absent.) | `golden_sample16k.json` (`assets/resources/sample_16k.wav`, English) | exact | exact | exact | 41/43 exact | | `golden_sample16k_bf16.json` (same audio, bf16 reference) | exact | exact | exact | 41/43 exact | -Everything a client observes is identical to the reference on every clip: the -offline transcript, the committed delta stream (including the reference's -`language` metadata-fragment artifact), and the final transcript. +These are the original port's reference-parity results. The regression +comparison now filters metadata-only prefixes from the unmodified Python +goldens before checking committed deltas and per-chunk `fixed_text`. The C++ +stream intentionally excludes the reference's `language` artifact; offline +and final transcript expectations are unchanged. ### The two English chunks that differ internally @@ -337,5 +332,4 @@ port logic: delta and the final transcript is identical. So per-chunk `fixed_text` equality is exact for the Chinese reference clip and -stable to within one token for long English audio, while every observable -output is exact. +stable to within one token for long English audio, with metadata artifacts excluded from committed output. diff --git a/src/community_models/r2t2_asr/session.cpp b/src/community_models/r2t2_asr/session.cpp index 88d031b37..9be0be6a9 100644 --- a/src/community_models/r2t2_asr/session.cpp +++ b/src/community_models/r2t2_asr/session.cpp @@ -366,6 +366,10 @@ R2T2ASRSession::StreamOutcome R2T2ASRSession::decode_stream_chunk(bool final_flu std::string fixed_text = decode_rollback_prefix(current_ids, k); if (contains_asr_text_tag(fixed_text)) { fixed_text = text_after_asr_tag(fixed_text); + } else if (force_language_.empty()) { + // Rollback may remove the separator even when raw_decoded_ has it. + // Until the stable prefix reaches , it is only metadata. + fixed_text.clear(); } fixed_text = truncate_at_pipe(fixed_text); @@ -394,16 +398,9 @@ R2T2ASRSession::StreamOutcome R2T2ASRSession::decode_stream_chunk(bool final_flu } void R2T2ASRSession::publish_stream_delta(const std::string & fixed_text, runtime::StreamEvent & event) { - // Mirrors the reference WebSocket integrator, which slices the committed - // text by the previously published length (in code points): - // - // if len(fixed) > len(last_fixed): emit fixed[len(last_fixed):] - // - // The stable prefix can regress between chunks (for example a token - // rollback can leave a partial metadata fragment such as "language"), and - // the reference does not rewrite what it already sent. The authoritative - // transcript is delivered in the final result, so consumers that need exact - // text use that. + // fixed_text contains transcript text only; metadata must never advance + // this code-point offset. Stable transcript prefixes may still shrink + // between chunks, so only publish newly committed code points. const size_t length = utf8_codepoint_count(fixed_text); if (length <= published_codepoints_) { return; diff --git a/tests/r2t2_asr/compare.py b/tests/r2t2_asr/compare.py index 1f4908508..4a496a51b 100644 --- a/tests/r2t2_asr/compare.py +++ b/tests/r2t2_asr/compare.py @@ -77,6 +77,17 @@ def transcript_from_stdout(stdout): return None +def transcript_fixed_text(chunk, language): + """Sanitize reference metadata fragments without changing the raw golden.""" + fixed = chunk["fixed_text"] + if not language or language.lower() == "auto": + raw = chunk["raw_decoded"] + metadata = raw.split("", 1)[0] + if "" not in raw or (fixed and metadata.startswith(fixed)): + return "" + return fixed + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--cli", required=True) @@ -131,11 +142,10 @@ def main(): got_chunks = parse_trace(trace_path) got_final = transcript_from_stdout(streams.stdout) - # Observable committed stream: the reference WebSocket integrator emits - # fixed_text[len(last):] whenever the stable prefix grows (code points). + # Match transcript-only stable prefixes, excluding reference metadata artifacts. last, expected_stream = "", [] for chunk in golden["stream_chunks"]: - fixed = chunk["fixed_text"] + fixed = transcript_fixed_text(chunk, golden.get("language")) if len(fixed) > len(last): expected_stream.append(fixed[len(last):]) last = fixed @@ -158,6 +168,7 @@ def main(): failures.append(("chunk-count", len(want_chunks), len(per_chunk))) print(f"chunk count differ: want {len(want_chunks)} got {len(per_chunk)}") for i, (want, got) in enumerate(zip(want_chunks, per_chunk)): + want = dict(want, fixed_text=transcript_fixed_text(want, golden.get("language"))) if got["fixed_text"] != want["fixed_text"]: failures.append((f"chunk[{i}].fixed_text", want["fixed_text"], got["fixed_text"])) print(f"chunk[{i}] fixed_text MISMATCH\n want: {want['fixed_text']!r}\n got: {got['fixed_text']!r}") diff --git a/tests/r2t2_asr/test_r2t2_asr_transcription.cpp b/tests/r2t2_asr/test_r2t2_asr_transcription.cpp index 0273a0add..6d24878cf 100644 --- a/tests/r2t2_asr/test_r2t2_asr_transcription.cpp +++ b/tests/r2t2_asr/test_r2t2_asr_transcription.cpp @@ -109,6 +109,7 @@ std::string run_streaming( committed += event.partial_text->text; } }); + request.options["language"] = "Auto"; streaming->start_stream(request); const int64_t chunk_frames = std::max( @@ -130,6 +131,11 @@ std::string run_streaming( throw std::runtime_error("streaming run produced no text"); } std::cout << "Committed stream: " << committed << "\n"; + // finish_stream() returns the final tail separately; the emitted deltas + // must form a nonempty transcript prefix, never consume metadata offsets. + if (committed.empty() || std::string(kExpectedStreamFinal).compare(0, committed.size(), committed) != 0) { + throw std::runtime_error("committed deltas are not a prefix of the expected transcript: " + committed); + } return result.text_output->text; } From a92a36f909fc54d5851755373258b2c01017aa00 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 11:29:34 +0900 Subject: [PATCH 3/9] refactor(r2t2): rename family to confucius4_r2t2 --- CMakeLists.txt | 36 +++--- app/server/example.json | 8 +- docs/asr.md | 8 +- docs/community_models/r2t2.md | 62 +++++----- .../{r2t2_asr => confucius4_r2t2}/assets.h | 10 +- .../audio_encoder.h | 8 +- .../frontend_whisper.h | 8 +- .../prompt_asr.h | 8 +- .../{r2t2_asr => confucius4_r2t2}/session.h | 20 ++-- .../text_postprocess.h | 4 +- .../{r2t2_asr => confucius4_r2t2}/thinker.h | 8 +- .../tokenizer_text.h | 8 +- .../{r2t2_asr => confucius4_r2t2}/types.h | 12 +- .../{r2t2_asr.json => confucius4_r2t2.json} | 10 +- .../{r2t2_asr => confucius4_r2t2}/assets.cpp | 14 +-- .../audio_encoder.cpp | 28 ++--- .../frontend_whisper.cpp | 12 +- .../prompt_asr.cpp | 6 +- .../{r2t2_asr => confucius4_r2t2}/session.cpp | 110 +++++++++--------- .../text_postprocess.cpp | 6 +- .../{r2t2_asr => confucius4_r2t2}/thinker.cpp | 6 +- .../tokenizer_text.cpp | 6 +- tests/{r2t2_asr => confucius4_r2t2}/README.md | 14 +-- .../{r2t2_asr => confucius4_r2t2}/compare.py | 26 ++--- .../{r2t2_asr => confucius4_r2t2}/golden.json | 0 .../golden_sample16k.json | 0 .../golden_sample16k_bf16.json | 0 .../golden_zh.json | 0 .../make_golden.py | 4 +- .../test_confucius4_r2t2_transcription.cpp} | 18 +-- webui/configs/model_params.json | 12 +- webui/configs/models_catalog.json | 2 +- webui/native/dist/index.html | 12 +- webui/native/src/lib/catalog.ts | 2 +- webui/native/src/routes/+page.svelte | 6 +- 35 files changed, 247 insertions(+), 247 deletions(-) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/assets.h (86%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/audio_encoder.h (78%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/frontend_whisper.h (62%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/prompt_asr.h (54%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/session.h (88%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/text_postprocess.h (96%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/thinker.h (82%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/tokenizer_text.h (83%) rename include/engine/community_models/{r2t2_asr => confucius4_r2t2}/types.h (76%) rename model_specs/{r2t2_asr.json => confucius4_r2t2.json} (97%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/assets.cpp (96%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/audio_encoder.cpp (95%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/frontend_whisper.cpp (85%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/prompt_asr.cpp (64%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/session.cpp (85%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/text_postprocess.cpp (99%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/thinker.cpp (96%) rename src/community_models/{r2t2_asr => confucius4_r2t2}/tokenizer_text.cpp (95%) rename tests/{r2t2_asr => confucius4_r2t2}/README.md (75%) rename tests/{r2t2_asr => confucius4_r2t2}/compare.py (88%) rename tests/{r2t2_asr => confucius4_r2t2}/golden.json (100%) rename tests/{r2t2_asr => confucius4_r2t2}/golden_sample16k.json (100%) rename tests/{r2t2_asr => confucius4_r2t2}/golden_sample16k_bf16.json (100%) rename tests/{r2t2_asr => confucius4_r2t2}/golden_zh.json (100%) rename tests/{r2t2_asr => confucius4_r2t2}/make_golden.py (96%) rename tests/{r2t2_asr/test_r2t2_asr_transcription.cpp => confucius4_r2t2/test_confucius4_r2t2_transcription.cpp} (91%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 708f40034..fef666dd2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1469,20 +1469,20 @@ audiocpp_add_model(qwen3_asr qwen3_forced_aligner ) -audiocpp_add_model(r2t2_asr +audiocpp_add_model(confucius4_r2t2 SOURCES - src/community_models/r2t2_asr/assets.cpp - src/community_models/r2t2_asr/text_postprocess.cpp - src/community_models/r2t2_asr/tokenizer_text.cpp - src/community_models/r2t2_asr/frontend_whisper.cpp - src/community_models/r2t2_asr/audio_encoder.cpp - src/community_models/r2t2_asr/thinker.cpp - src/community_models/r2t2_asr/prompt_asr.cpp - src/community_models/r2t2_asr/session.cpp + src/community_models/confucius4_r2t2/assets.cpp + src/community_models/confucius4_r2t2/text_postprocess.cpp + src/community_models/confucius4_r2t2/tokenizer_text.cpp + src/community_models/confucius4_r2t2/frontend_whisper.cpp + src/community_models/confucius4_r2t2/audio_encoder.cpp + src/community_models/confucius4_r2t2/thinker.cpp + src/community_models/confucius4_r2t2/prompt_asr.cpp + src/community_models/confucius4_r2t2/session.cpp INCLUDES - engine/community_models/r2t2_asr/session.h + engine/community_models/confucius4_r2t2/session.h LOADERS - engine::community_models::r2t2_asr::make_r2t2_asr_loader + engine::community_models::confucius4_r2t2::make_confucius4_r2t2_loader ) audiocpp_add_model(qwen3_forced_aligner @@ -2987,17 +2987,17 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST target_link_libraries(test_granite5asr_golden_transcription PRIVATE OpenMP::OpenMP_CXX) endif() - if (r2t2_asr IN_LIST AUDIOCPP_LINKED_MODELS) - add_executable(test_r2t2_asr_transcription - tests/r2t2_asr/test_r2t2_asr_transcription.cpp + if (confucius4_r2t2 IN_LIST AUDIOCPP_LINKED_MODELS) + add_executable(test_confucius4_r2t2_transcription + tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp ) - target_compile_definitions(test_r2t2_asr_transcription PRIVATE + target_compile_definitions(test_confucius4_r2t2_transcription PRIVATE ENGINE_REPO_ROOT="${CMAKE_CURRENT_SOURCE_DIR}" ) - target_link_libraries(test_r2t2_asr_transcription PRIVATE engine_runtime ggml) - target_include_directories(test_r2t2_asr_transcription PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(test_confucius4_r2t2_transcription PRIVATE engine_runtime ggml) + target_include_directories(test_confucius4_r2t2_transcription PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) if (ENGINE_ENABLE_OPENMP) - target_link_libraries(test_r2t2_asr_transcription PRIVATE OpenMP::OpenMP_CXX) + target_link_libraries(test_confucius4_r2t2_transcription PRIVATE OpenMP::OpenMP_CXX) endif() endif() diff --git a/app/server/example.json b/app/server/example.json index 9ac63be64..7b2e73350 100644 --- a/app/server/example.json +++ b/app/server/example.json @@ -39,20 +39,20 @@ }, { "id": "r2t2-asr", - "family": "r2t2_asr", + "family": "confucius4_r2t2", "path": "../../models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", "task": "asr", "mode": "offline" }, { "id": "r2t2-asr-stream", - "family": "r2t2_asr", + "family": "confucius4_r2t2", "path": "../../models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", "task": "asr", "mode": "streaming", "session_options": { - "r2t2_asr.chunk_size_ms": "320", - "r2t2_asr.max_tokens": "32" + "confucius4_r2t2.chunk_size_ms": "320", + "confucius4_r2t2.max_tokens": "32" } } ] diff --git a/docs/asr.md b/docs/asr.md index 5b286567a..42e73efcc 100644 --- a/docs/asr.md +++ b/docs/asr.md @@ -7,7 +7,7 @@ | Fun-ASR-Nano | `fun_asr_nano` | offline | [Fun-ASR-Nano](#fun-asr-nano) | | Granite Speech 5.0 TurboCTC | `granite5asr` | offline | [Granite Speech 5.0 TurboCTC](community_models/granite5asr.md) | | Qwen3 ASR | `qwen3_asr` | offline, streaming | [Qwen3 ASR](#qwen3-asr) | -| Confucius4-R2T2 | `r2t2_asr` | offline, streaming | [Confucius4-R2T2](community_models/r2t2.md) | +| Confucius4-R2T2 | `confucius4_r2t2` | offline, streaming | [Confucius4-R2T2](community_models/r2t2.md) | | Citrinet ASR | `citrinet_asr` | offline | [Citrinet ASR](#citrinet-asr) | | Kroko Community ASR | `kroko_asr` | offline, streaming | [Kroko Community ASR](#kroko-community-asr) | | Higgs Audio STT | `higgs_audio_stt` | offline, streaming | [Higgs Audio STT](models/higgs_audio_stt.md) | @@ -70,14 +70,14 @@ never revised, and chunk sizes from 80 ms to 2 s are supported. It runs the same audio tower as Qwen3 ASR, so only the streaming state machine differs. ```bash -audiocpp_cli --task asr --family r2t2_asr --model models/Confucius4-R2T2 \ +audiocpp_cli --task asr --family confucius4_r2t2 --model models/Confucius4-R2T2 \ --backend metal --audio speech_16k.wav --text-out transcript.txt ``` ```bash -audiocpp_cli --task asr --mode streaming --family r2t2_asr \ +audiocpp_cli --task asr --mode streaming --family confucius4_r2t2 \ --model models/Confucius4-R2T2 --backend metal --audio speech_16k.wav \ - --session-option r2t2_asr.chunk_size_ms=320 --text-out transcript.txt + --session-option confucius4_r2t2.chunk_size_ms=320 --text-out transcript.txt ``` Streaming emits append-only partial text; the final transcript is returned when diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index aeb62bdbf..924d03a2e 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -10,7 +10,7 @@ which is what makes it suitable for live captioning and downstream agents. | Field | Value | |---|---| -| Family | `r2t2_asr` | +| Family | `confucius4_r2t2` | | HF checkpoint | `netease-youdao/Confucius4-R2T2` | | Task | `asr` | | Modes | `offline`, `streaming` | @@ -22,15 +22,15 @@ which is what makes it suitable for live captioning and downstream agents. ## Independent implementation -`r2t2_asr` is a **standalone family** (a community port, `status: community` in +`confucius4_r2t2` is a **standalone family** (a community port, `status: community` in the spec): assets, Whisper log-mel frontend, windowed audio tower, tokenizer, LSP session, and text post-processing live under -`src/community_models/r2t2_asr/` + `include/engine/community_models/r2t2_asr/` and +`src/community_models/confucius4_r2t2/` + `include/engine/community_models/confucius4_r2t2/` and share no code with the `qwen3_asr` family, so the two can evolve independently. The loader itself is the framework's **schema-v1 spec-backed loader** — -`model_specs/r2t2_asr.json` is the single source of truth for metadata, +`model_specs/confucius4_r2t2.json` is the single source of truth for metadata, capabilities, options and packages, and the factory lives next to the session -(`make_r2t2_asr_loader`), so there is no per-model `loader.{h,cpp}`. What it does +(`make_confucius4_r2t2_loader`), so there is no per-model `loader.{h,cpp}`. What it does **not** duplicate is framework infrastructure: * the thinker is a thin adapter over the shared greedy Qwen decoder runtime @@ -60,12 +60,12 @@ The R2T2 config declares interleaved mrope with `mrope_section [24, 20, 20]`, bu the model only ever sees audio and text, so all three position streams are identical and mrope degenerates to standard NEOX RoPE. The shared decoder's NEOX rope therefore reproduces the reference numerics, which the golden checks -in `tests/r2t2_asr/` confirm chunk by chunk. +in `tests/confucius4_r2t2/` confirm chunk by chunk. ## Install ```bash -python3 tools/model_manager_v2.py install r2t2_asr_safetensors +python3 tools/model_manager_v2.py install confucius4_r2t2_safetensors ``` Or point `--model` at any directory with the HF checkpoint layout (`config.json`, @@ -78,7 +78,7 @@ tensor path. ## Offline transcription ```bash -audiocpp_cli --task asr --family r2t2_asr \ +audiocpp_cli --task asr --family confucius4_r2t2 \ --model models/Confucius4-R2T2 --backend metal \ --audio speech_16k.wav --text-out transcript.txt ``` @@ -91,10 +91,10 @@ Chinese` (or any supported language) to skip language detection, and `--text ## Streaming transcription ```bash -audiocpp_cli --task asr --mode streaming --family r2t2_asr \ +audiocpp_cli --task asr --mode streaming --family confucius4_r2t2 \ --model models/Confucius4-R2T2 --backend metal \ --audio speech_16k.wav \ - --session-option r2t2_asr.chunk_size_ms=320 \ + --session-option confucius4_r2t2.chunk_size_ms=320 \ --text-out transcript.txt ``` @@ -111,17 +111,17 @@ quality. | Option | Values | Default | Meaning | |---|---|---:|---| -| `r2t2_asr.chunk_size_ms` | 80-2000 | `320` | Streaming decode chunk in milliseconds. | -| `r2t2_asr.unfixed_chunk_num` | integer | `2` | Leading chunks decoded without a stable-prefix prompt. | -| `r2t2_asr.unfixed_token_num` | integer | `5` | Tokens rolled back from the accumulated text before it is used as the prefix prompt. | -| `r2t2_asr.rollback_punctuation` | `true`, `false` | `false` | Keep trailing text uncommitted when it already ends with punctuation instead of rolling back tokens. | -| `r2t2_asr.max_tokens` | integer | `32` | Greedy decode budget per chunk and for the final flush. | -| `r2t2_asr.audio_encoder_weight_type` | `native`, `f32`, `f16` | `native` | Audio tower weight storage. | -| `r2t2_asr.thinker_weight_type` (alias `r2t2_asr.weight_type`) | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Thinker weight storage. | -| `r2t2_asr.audio_encoder_graph_arena_mb` | MB | `128` | Audio tower graph arena. | -| `r2t2_asr.thinker_prefill_graph_arena_mb` | MB | `256` | Thinker prefill graph arena. | -| `r2t2_asr.thinker_decode_graph_arena_mb` | MB | `256` | Thinker decode graph arena. | -| `r2t2_asr.thinker_weight_context_mb` | MB | `64` | Thinker weight context. | +| `confucius4_r2t2.chunk_size_ms` | 80-2000 | `320` | Streaming decode chunk in milliseconds. | +| `confucius4_r2t2.unfixed_chunk_num` | integer | `2` | Leading chunks decoded without a stable-prefix prompt. | +| `confucius4_r2t2.unfixed_token_num` | integer | `5` | Tokens rolled back from the accumulated text before it is used as the prefix prompt. | +| `confucius4_r2t2.rollback_punctuation` | `true`, `false` | `false` | Keep trailing text uncommitted when it already ends with punctuation instead of rolling back tokens. | +| `confucius4_r2t2.max_tokens` | integer | `32` | Greedy decode budget per chunk and for the final flush. | +| `confucius4_r2t2.audio_encoder_weight_type` | `native`, `f32`, `f16` | `native` | Audio tower weight storage. | +| `confucius4_r2t2.thinker_weight_type` (alias `confucius4_r2t2.weight_type`) | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Thinker weight storage. | +| `confucius4_r2t2.audio_encoder_graph_arena_mb` | MB | `128` | Audio tower graph arena. | +| `confucius4_r2t2.thinker_prefill_graph_arena_mb` | MB | `256` | Thinker prefill graph arena. | +| `confucius4_r2t2.thinker_decode_graph_arena_mb` | MB | `256` | Thinker decode graph arena. | +| `confucius4_r2t2.thinker_weight_context_mb` | MB | `64` | Thinker weight context. | ### Request options (use with `--request-option`) @@ -208,11 +208,11 @@ model license — see the repository's `LICENSE`, `LICENSE_zh`, and `NOTICE`): | Package id | File | Quantization | Size | |---|---|---|---:| -| `r2t2_asr_q8_0` (default) | `r2t2-q8_0.gguf` | Q8_0 | 2.31 GiB | -| `r2t2_asr_f16` | `r2t2-f16.gguf` | F16 | 3.81 GiB | +| `confucius4_r2t2_q8_0` (default) | `r2t2-q8_0.gguf` | Q8_0 | 2.31 GiB | +| `confucius4_r2t2_f16` | `r2t2-f16.gguf` | F16 | 3.81 GiB | ```bash -python3 tools/model_manager_v2.py install r2t2_asr_q8_0 # or r2t2_asr_f16 +python3 tools/model_manager_v2.py install confucius4_r2t2_q8_0 # or confucius4_r2t2_f16 ``` These files are self-contained: tokenizer, processor/generation config, chat @@ -229,7 +229,7 @@ Convert a real (non-symlinked) checkpoint directory: audiocpp_gguf \ --input /path/to/Confucius4-R2T2/model.safetensors \ --root /path/to/Confucius4-R2T2 \ - --family r2t2_asr \ + --family confucius4_r2t2 \ --output models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf \ --type q8_0 ``` @@ -241,7 +241,7 @@ snapshot is supported and does not need this step. The output embeds the sidecars and the model spec, so the single `.gguf` is portable: ```bash -audiocpp_cli --task asr --family r2t2_asr \ +audiocpp_cli --task asr --family confucius4_r2t2 \ --model models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf --backend metal \ --audio speech_16k.wav ``` @@ -273,18 +273,18 @@ The port is verified against the macOS MPS reference with golden traces: ```bash # 1. Golden from the reference implementation (in the Confucius4-R2T2 repo) cd /path/to/Confucius4-R2T2 -PYTHONPATH=. uv run python /path/to/audio.cpp/tests/r2t2_asr/make_golden.py \ +PYTHONPATH=. uv run python /path/to/audio.cpp/tests/confucius4_r2t2/make_golden.py \ --model_path /path/to/audio.cpp/models/Confucius4-R2T2 \ --audio resources/test.wav \ - --out /path/to/audio.cpp/tests/r2t2_asr/golden.json + --out /path/to/audio.cpp/tests/confucius4_r2t2/golden.json # 2. Compare the C++ runtime (offline text, per-chunk committed text, final text) cd /path/to/audio.cpp -python3 tests/r2t2_asr/compare.py \ +python3 tests/confucius4_r2t2/compare.py \ --cli build/macos-metal-release/bin/audiocpp_cli \ --model models/Confucius4-R2T2 \ --audio /tmp/test.wav \ - --golden tests/r2t2_asr/golden.json --backend metal + --golden tests/confucius4_r2t2/golden.json --backend metal ``` `compare.py` runs the CLI with trace logging, parses the per-chunk trace, and @@ -292,7 +292,7 @@ diffs it against the golden. A repo-native smoke test runs both paths without the Python environment: ```bash -build/macos-metal-release/bin/test_r2t2_asr_transcription --backend metal +build/macos-metal-release/bin/test_confucius4_r2t2_transcription --backend metal ``` (It skips with exit code 125 when `models/Confucius4-R2T2` or the audio asset is diff --git a/include/engine/community_models/r2t2_asr/assets.h b/include/engine/community_models/confucius4_r2t2/assets.h similarity index 86% rename from include/engine/community_models/r2t2_asr/assets.h rename to include/engine/community_models/confucius4_r2t2/assets.h index f844e1188..33a1bfd04 100644 --- a/include/engine/community_models/r2t2_asr/assets.h +++ b/include/engine/community_models/confucius4_r2t2/assets.h @@ -13,7 +13,7 @@ namespace engine::assets { class TensorSource; } -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { struct R2T2ASRAudioEncoderConfig { int64_t num_mel_bins = 128; @@ -81,11 +81,11 @@ struct R2T2ASRAssets { std::shared_ptr model_weights; }; -std::shared_ptr load_r2t2_asr_assets(const std::filesystem::path & model_path); -std::shared_ptr load_r2t2_asr_assets( +std::shared_ptr load_confucius4_r2t2_assets(const std::filesystem::path & model_path); +std::shared_ptr load_confucius4_r2t2_assets( const std::filesystem::path & model_path, std::string_view package_family); -std::shared_ptr load_r2t2_asr_assets( +std::shared_ptr load_confucius4_r2t2_assets( assets::ResourceBundle resources); -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/audio_encoder.h b/include/engine/community_models/confucius4_r2t2/audio_encoder.h similarity index 78% rename from include/engine/community_models/r2t2_asr/audio_encoder.h rename to include/engine/community_models/confucius4_r2t2/audio_encoder.h index 7e70677ef..e125cb98e 100644 --- a/include/engine/community_models/r2t2_asr/audio_encoder.h +++ b/include/engine/community_models/confucius4_r2t2/audio_encoder.h @@ -2,13 +2,13 @@ #include "engine/framework/assets/tensor_source.h" #include "engine/framework/core/execution_context.h" -#include "engine/community_models/r2t2_asr/assets.h" -#include "engine/community_models/r2t2_asr/types.h" +#include "engine/community_models/confucius4_r2t2/assets.h" +#include "engine/community_models/confucius4_r2t2/types.h" #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { class R2T2ASRAudioEncoderGraph; struct R2T2ASRAudioEncoderWeights; @@ -32,4 +32,4 @@ class R2T2ASRAudioEncoderRuntime { std::unique_ptr graph_; }; -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/frontend_whisper.h b/include/engine/community_models/confucius4_r2t2/frontend_whisper.h similarity index 62% rename from include/engine/community_models/r2t2_asr/frontend_whisper.h rename to include/engine/community_models/confucius4_r2t2/frontend_whisper.h index 713183301..9b5657789 100644 --- a/include/engine/community_models/r2t2_asr/frontend_whisper.h +++ b/include/engine/community_models/confucius4_r2t2/frontend_whisper.h @@ -1,12 +1,12 @@ #pragma once #include "engine/framework/audio/dsp.h" -#include "engine/community_models/r2t2_asr/assets.h" -#include "engine/community_models/r2t2_asr/types.h" +#include "engine/community_models/confucius4_r2t2/assets.h" +#include "engine/community_models/confucius4_r2t2/types.h" #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { class R2T2ASRWhisperFrontend { public: @@ -19,4 +19,4 @@ class R2T2ASRWhisperFrontend { engine::audio::WhisperLogMelExtractor extractor_; }; -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/prompt_asr.h b/include/engine/community_models/confucius4_r2t2/prompt_asr.h similarity index 54% rename from include/engine/community_models/r2t2_asr/prompt_asr.h rename to include/engine/community_models/confucius4_r2t2/prompt_asr.h index cbc5a8ddc..41882fefc 100644 --- a/include/engine/community_models/r2t2_asr/prompt_asr.h +++ b/include/engine/community_models/confucius4_r2t2/prompt_asr.h @@ -1,9 +1,9 @@ #pragma once -#include "engine/community_models/r2t2_asr/tokenizer_text.h" -#include "engine/community_models/r2t2_asr/types.h" +#include "engine/community_models/confucius4_r2t2/tokenizer_text.h" +#include "engine/community_models/confucius4_r2t2/types.h" -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { class R2T2ASRPromptBuilder { public: @@ -15,4 +15,4 @@ class R2T2ASRPromptBuilder { const R2T2ASRTextTokenizer & tokenizer_; }; -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/session.h b/include/engine/community_models/confucius4_r2t2/session.h similarity index 88% rename from include/engine/community_models/r2t2_asr/session.h rename to include/engine/community_models/confucius4_r2t2/session.h index dc176c791..1e9d319da 100644 --- a/include/engine/community_models/r2t2_asr/session.h +++ b/include/engine/community_models/confucius4_r2t2/session.h @@ -2,12 +2,12 @@ #include "engine/framework/runtime/session_base.h" #include "engine/framework/runtime/model.h" -#include "engine/community_models/r2t2_asr/assets.h" -#include "engine/community_models/r2t2_asr/audio_encoder.h" -#include "engine/community_models/r2t2_asr/frontend_whisper.h" -#include "engine/community_models/r2t2_asr/thinker.h" -#include "engine/community_models/r2t2_asr/tokenizer_text.h" -#include "engine/community_models/r2t2_asr/types.h" +#include "engine/community_models/confucius4_r2t2/assets.h" +#include "engine/community_models/confucius4_r2t2/audio_encoder.h" +#include "engine/community_models/confucius4_r2t2/frontend_whisper.h" +#include "engine/community_models/confucius4_r2t2/thinker.h" +#include "engine/community_models/confucius4_r2t2/tokenizer_text.h" +#include "engine/community_models/confucius4_r2t2/types.h" #include #include @@ -15,12 +15,12 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { /// Spec-backed loader factory (schema-v1 contract): the framework derives -/// metadata, capabilities and option validation from model_specs/r2t2_asr.json, +/// metadata, capabilities and option validation from model_specs/confucius4_r2t2.json, /// so this family ships no per-model loader.{h,cpp}. -std::shared_ptr make_r2t2_asr_loader(); +std::shared_ptr make_confucius4_r2t2_loader(); /// Streaming decode configuration; defaults mirror /// R2T2ASRModel.init_streaming_state() in the reference implementation. @@ -120,4 +120,4 @@ class R2T2ASRSession final std::chrono::steady_clock::time_point stream_wall_start_{}; }; -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/text_postprocess.h b/include/engine/community_models/confucius4_r2t2/text_postprocess.h similarity index 96% rename from include/engine/community_models/r2t2_asr/text_postprocess.h rename to include/engine/community_models/confucius4_r2t2/text_postprocess.h index fc2b11a5a..dc0f525b7 100644 --- a/include/engine/community_models/r2t2_asr/text_postprocess.h +++ b/include/engine/community_models/confucius4_r2t2/text_postprocess.h @@ -4,7 +4,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { // Faithful C++ ports of the R2T2 text post-processing pipeline // (r2t2/r2t2_asr.py and qwen_asr/inference/utils.py). All functions operate @@ -74,4 +74,4 @@ std::size_t utf8_codepoint_count(const std::string & text); /// integrator's `fixed_text[len(last_fixed_text):]` slice. std::string utf8_slice_from_codepoint(const std::string & text, std::size_t start_codepoint); -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/thinker.h b/include/engine/community_models/confucius4_r2t2/thinker.h similarity index 82% rename from include/engine/community_models/r2t2_asr/thinker.h rename to include/engine/community_models/confucius4_r2t2/thinker.h index dd73cd896..529051088 100644 --- a/include/engine/community_models/r2t2_asr/thinker.h +++ b/include/engine/community_models/confucius4_r2t2/thinker.h @@ -2,13 +2,13 @@ #include "engine/framework/assets/tensor_source.h" #include "engine/framework/core/execution_context.h" -#include "engine/community_models/r2t2_asr/assets.h" -#include "engine/community_models/r2t2_asr/types.h" +#include "engine/community_models/confucius4_r2t2/assets.h" +#include "engine/community_models/confucius4_r2t2/types.h" #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { /// Greedy decoding of the Confucius4-R2T2 thinker. The Qwen3-style decoder /// stack, the audio-embedding injection, the static KV cache, and the graph @@ -37,4 +37,4 @@ class R2T2ASRThinkerRuntime { std::unique_ptr impl_; }; -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/tokenizer_text.h b/include/engine/community_models/confucius4_r2t2/tokenizer_text.h similarity index 83% rename from include/engine/community_models/r2t2_asr/tokenizer_text.h rename to include/engine/community_models/confucius4_r2t2/tokenizer_text.h index 32c11f467..b7ebb7ffd 100644 --- a/include/engine/community_models/r2t2_asr/tokenizer_text.h +++ b/include/engine/community_models/confucius4_r2t2/tokenizer_text.h @@ -1,13 +1,13 @@ #pragma once -#include "engine/community_models/r2t2_asr/assets.h" -#include "engine/community_models/r2t2_asr/types.h" +#include "engine/community_models/confucius4_r2t2/assets.h" +#include "engine/community_models/confucius4_r2t2/types.h" #include #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { class R2T2ASRTextTokenizer { public: @@ -40,4 +40,4 @@ class R2T2ASRTextTokenizer { std::shared_ptr impl_; }; -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/include/engine/community_models/r2t2_asr/types.h b/include/engine/community_models/confucius4_r2t2/types.h similarity index 76% rename from include/engine/community_models/r2t2_asr/types.h rename to include/engine/community_models/confucius4_r2t2/types.h index fe116ef46..6ed71eac7 100644 --- a/include/engine/community_models/r2t2_asr/types.h +++ b/include/engine/community_models/confucius4_r2t2/types.h @@ -7,7 +7,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { struct R2T2ASRGenerationOptions { int64_t max_new_tokens = 512; @@ -52,7 +52,7 @@ struct R2T2ASRGeneratedTokens { std::vector token_ids; }; -inline int64_t r2t2_asr_floor_div(int64_t numerator, int64_t denominator) { +inline int64_t confucius4_r2t2_floor_div(int64_t numerator, int64_t denominator) { int64_t quotient = numerator / denominator; const int64_t remainder = numerator % denominator; if (remainder != 0 && ((remainder < 0) != (denominator < 0))) { @@ -61,14 +61,14 @@ inline int64_t r2t2_asr_floor_div(int64_t numerator, int64_t denominator) { return quotient; } -inline int64_t r2t2_asr_audio_encoder_token_count(int64_t input_frames) { +inline int64_t confucius4_r2t2_audio_encoder_token_count(int64_t input_frames) { if (input_frames <= 0) { throw std::runtime_error("R2T2 ASR requires positive feature frame count"); } const int64_t input_lengths_leave = input_frames % 100; - const int64_t feat_lengths = r2t2_asr_floor_div(input_lengths_leave - 1, 2) + 1; - return r2t2_asr_floor_div(r2t2_asr_floor_div(feat_lengths - 1, 2) + 1 - 1, 2) + 1 + + const int64_t feat_lengths = confucius4_r2t2_floor_div(input_lengths_leave - 1, 2) + 1; + return confucius4_r2t2_floor_div(confucius4_r2t2_floor_div(feat_lengths - 1, 2) + 1 - 1, 2) + 1 + (input_frames / 100) * 13; } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/model_specs/r2t2_asr.json b/model_specs/confucius4_r2t2.json similarity index 97% rename from model_specs/r2t2_asr.json rename to model_specs/confucius4_r2t2.json index c5fd19adc..fdf8b33af 100644 --- a/model_specs/r2t2_asr.json +++ b/model_specs/confucius4_r2t2.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "family": "r2t2_asr", + "family": "confucius4_r2t2", "display_name": "Confucius4-R2T2", "description": "NetEase Youdao Confucius4-R2T2 real-time ASR: a Qwen3-ASR fine-tune with Longest Stable Prefix (LSP) streaming. Low-latency append-only streaming from 80 ms to 2 s chunks, context and hotword prompts, 30 languages.", "category": "asr", @@ -56,7 +56,7 @@ ] }, "ui": { - "recommended_package": "r2t2_asr_q8_0", + "recommended_package": "confucius4_r2t2_q8_0", "tags": [ "ASR", "Stream" @@ -76,7 +76,7 @@ }, "packages": [ { - "id": "r2t2_asr_q8_0", + "id": "confucius4_r2t2_q8_0", "display_name": "Confucius4-R2T2 Q8_0 GGUF", "default": true, "format": "gguf", @@ -91,7 +91,7 @@ } }, { - "id": "r2t2_asr_f16", + "id": "confucius4_r2t2_f16", "display_name": "Confucius4-R2T2 F16 GGUF", "format": "gguf", "precision": "f16", @@ -105,7 +105,7 @@ } }, { - "id": "r2t2_asr_safetensors", + "id": "confucius4_r2t2_safetensors", "display_name": "Confucius4-R2T2 (HF safetensors)", "format": "safetensors", "precision": "native", diff --git a/src/community_models/r2t2_asr/assets.cpp b/src/community_models/confucius4_r2t2/assets.cpp similarity index 96% rename from src/community_models/r2t2_asr/assets.cpp rename to src/community_models/confucius4_r2t2/assets.cpp index 5f20eb693..9f0b80ba0 100644 --- a/src/community_models/r2t2_asr/assets.cpp +++ b/src/community_models/confucius4_r2t2/assets.cpp @@ -1,4 +1,4 @@ -#include "engine/community_models/r2t2_asr/assets.h" +#include "engine/community_models/confucius4_r2t2/assets.h" #include "engine/framework/model_spec/package.h" #include "engine/framework/io/filesystem.h" @@ -12,7 +12,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { namespace json = engine::io::json; namespace { @@ -262,19 +262,19 @@ std::shared_ptr make_assets( } // namespace -std::shared_ptr load_r2t2_asr_assets(const std::filesystem::path & model_path) { - return load_r2t2_asr_assets(model_path, "r2t2_asr"); +std::shared_ptr load_confucius4_r2t2_assets(const std::filesystem::path & model_path) { + return load_confucius4_r2t2_assets(model_path, "confucius4_r2t2"); } -std::shared_ptr load_r2t2_asr_assets( +std::shared_ptr load_confucius4_r2t2_assets( const std::filesystem::path & model_path, std::string_view package_family) { return make_assets(make_resource_bundle(model_path, package_family)); } -std::shared_ptr load_r2t2_asr_assets( +std::shared_ptr load_confucius4_r2t2_assets( assets::ResourceBundle resources) { return make_assets(std::move(resources)); } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/src/community_models/r2t2_asr/audio_encoder.cpp b/src/community_models/confucius4_r2t2/audio_encoder.cpp similarity index 95% rename from src/community_models/r2t2_asr/audio_encoder.cpp rename to src/community_models/confucius4_r2t2/audio_encoder.cpp index 64141dba6..787913b01 100644 --- a/src/community_models/r2t2_asr/audio_encoder.cpp +++ b/src/community_models/confucius4_r2t2/audio_encoder.cpp @@ -1,4 +1,4 @@ -#include "engine/community_models/r2t2_asr/audio_encoder.h" +#include "engine/community_models/confucius4_r2t2/audio_encoder.h" #include "engine/framework/assets/tensor_source.h" #include "engine/framework/core/backend.h" @@ -21,7 +21,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { namespace assets = engine::assets; namespace modules = engine::modules; @@ -244,7 +244,7 @@ std::shared_ptr load_weights( auto store = std::make_shared( backend, backend_type, - "r2t2_asr.audio_encoder.weights", + "confucius4_r2t2.audio_encoder.weights", kDefaultAudioWeightContextBytes); weights->store = store; weights->conv1 = load_conv2d( @@ -377,7 +377,7 @@ class R2T2ASRAudioEncoderGraph { chunk_count_ = static_cast(chunk_lengths_.size()); chunk_token_lengths_.reserve(chunk_lengths_.size()); for (const int64_t chunk_length : chunk_lengths_) { - chunk_token_lengths_.push_back(r2t2_asr_audio_encoder_token_count(chunk_length)); + chunk_token_lengths_.push_back(confucius4_r2t2_audio_encoder_token_count(chunk_length)); } output_tokens_ = sum_values(chunk_token_lengths_); const int64_t max_chunk_tokens = max_value(chunk_token_lengths_); @@ -392,7 +392,7 @@ class R2T2ASRAudioEncoderGraph { throw std::runtime_error("failed to initialize R2T2 ASR audio encoder graph context"); } - core::ModuleBuildContext ctx{ctx_.get(), "r2t2_asr.audio_encoder", backend_type_}; + core::ModuleBuildContext ctx{ctx_.get(), "confucius4_r2t2.audio_encoder", backend_type_}; auto input = core::make_tensor( ctx, GGML_TYPE_F32, @@ -485,8 +485,8 @@ class R2T2ASRAudioEncoderGraph { throw std::runtime_error("failed to allocate R2T2 ASR audio encoder graph"); } ggml_backend_tensor_set(attention_mask_, attention_mask_values_.data(), 0, attention_mask_values_.size() * sizeof(float)); - debug::timing_log_scalar("r2t2_asr.audio_encoder.graph.build_ms", engine::debug::elapsed_ms(build_start, Clock::now())); - debug::trace_log_scalar("r2t2_asr.audio_encoder.frames", frames_); + debug::timing_log_scalar("confucius4_r2t2.audio_encoder.graph.build_ms", engine::debug::elapsed_ms(build_start, Clock::now())); + debug::trace_log_scalar("confucius4_r2t2.audio_encoder.frames", frames_); } ~R2T2ASRAudioEncoderGraph() { @@ -522,12 +522,12 @@ class R2T2ASRAudioEncoderGraph { auto timing_start = Clock::now(); ggml_backend_tensor_set(input_, padded_features.data(), 0, padded_features.size() * sizeof(float)); ggml_backend_tensor_set(attention_mask_, attention_mask_values_.data(), 0, attention_mask_values_.size() * sizeof(float)); - debug::timing_log_scalar("r2t2_asr.audio_encoder.input_upload_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); + debug::timing_log_scalar("confucius4_r2t2.audio_encoder.input_upload_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); core::set_backend_threads(backend_, compute_threads_); timing_start = Clock::now(); const ggml_status status = engine::core::compute_backend_graph(backend_, graph_); ggml_backend_synchronize(backend_); - debug::timing_log_scalar("r2t2_asr.audio_encoder.graph.compute_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); + debug::timing_log_scalar("confucius4_r2t2.audio_encoder.graph.compute_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); if (status != GGML_STATUS_SUCCESS) { throw std::runtime_error("R2T2 ASR audio encoder graph compute failed"); } @@ -537,7 +537,7 @@ class R2T2ASRAudioEncoderGraph { out.values.resize(static_cast(out.tokens * out.hidden_size)); timing_start = Clock::now(); ggml_backend_tensor_get(output_, out.values.data(), 0, out.values.size() * sizeof(float)); - debug::timing_log_scalar("r2t2_asr.audio_encoder.output_read_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); + debug::timing_log_scalar("confucius4_r2t2.audio_encoder.output_read_ms", engine::debug::elapsed_ms(timing_start, Clock::now())); return out; } @@ -589,7 +589,7 @@ R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeat if (execution_ == nullptr) { throw std::runtime_error("R2T2 ASR audio encoder execution context is null"); } - if (features.encoder_tokens != r2t2_asr_audio_encoder_token_count(features.frames)) { + if (features.encoder_tokens != confucius4_r2t2_audio_encoder_token_count(features.frames)) { throw std::runtime_error("R2T2 ASR audio encoder token count mismatch"); } const int threads = std::max(1, execution_->config().threads); @@ -602,8 +602,8 @@ R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeat graph_arena_bytes_, features.frames); } else { - debug::timing_log_scalar("r2t2_asr.audio_encoder.graph.build_ms", 0.0); - debug::trace_log_scalar("r2t2_asr.audio_encoder.frames", features.frames); + debug::timing_log_scalar("confucius4_r2t2.audio_encoder.graph.build_ms", 0.0); + debug::trace_log_scalar("confucius4_r2t2.audio_encoder.frames", features.frames); } auto out = graph_->run(features); if (out.tokens != features.encoder_tokens) { @@ -612,4 +612,4 @@ R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeat return out; } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/src/community_models/r2t2_asr/frontend_whisper.cpp b/src/community_models/confucius4_r2t2/frontend_whisper.cpp similarity index 85% rename from src/community_models/r2t2_asr/frontend_whisper.cpp rename to src/community_models/confucius4_r2t2/frontend_whisper.cpp index 8cbcf60a0..e1b670ee9 100644 --- a/src/community_models/r2t2_asr/frontend_whisper.cpp +++ b/src/community_models/confucius4_r2t2/frontend_whisper.cpp @@ -1,4 +1,4 @@ -#include "engine/community_models/r2t2_asr/frontend_whisper.h" +#include "engine/community_models/confucius4_r2t2/frontend_whisper.h" #include "engine/framework/audio/conversion.h" #include "engine/framework/audio/dsp.h" @@ -11,7 +11,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { namespace { constexpr int kMinInputSamples = 8000; @@ -84,10 +84,10 @@ R2T2ASRAudioFeatures R2T2ASRWhisperFrontend::extract(const runtime::AudioBuffer out.attention_mask.assign(static_cast(features.frames), 1); out.mel_bins = features.mel_bins; out.frames = features.frames; - out.encoder_tokens = r2t2_asr_audio_encoder_token_count(out.frames); - debug::timing_log_scalar("r2t2_asr.frontend.normalize_ms", engine::debug::elapsed_ms(normalize_start, normalize_end)); - debug::timing_log_scalar("r2t2_asr.frontend.log_mel_ms", engine::debug::elapsed_ms(feature_start, feature_end)); + out.encoder_tokens = confucius4_r2t2_audio_encoder_token_count(out.frames); + debug::timing_log_scalar("confucius4_r2t2.frontend.normalize_ms", engine::debug::elapsed_ms(normalize_start, normalize_end)); + debug::timing_log_scalar("confucius4_r2t2.frontend.log_mel_ms", engine::debug::elapsed_ms(feature_start, feature_end)); return out; } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/src/community_models/r2t2_asr/prompt_asr.cpp b/src/community_models/confucius4_r2t2/prompt_asr.cpp similarity index 64% rename from src/community_models/r2t2_asr/prompt_asr.cpp rename to src/community_models/confucius4_r2t2/prompt_asr.cpp index e82863893..eccdfcdfc 100644 --- a/src/community_models/r2t2_asr/prompt_asr.cpp +++ b/src/community_models/confucius4_r2t2/prompt_asr.cpp @@ -1,6 +1,6 @@ -#include "engine/community_models/r2t2_asr/prompt_asr.h" +#include "engine/community_models/confucius4_r2t2/prompt_asr.h" -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { R2T2ASRPromptBuilder::R2T2ASRPromptBuilder(const R2T2ASRTextTokenizer & tokenizer) : tokenizer_(tokenizer) {} @@ -9,4 +9,4 @@ R2T2ASRPrompt R2T2ASRPromptBuilder::build(const R2T2ASRRequest & request, int64_ return tokenizer_.build_prompt(request.context, request.language, audio_feature_tokens); } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/src/community_models/r2t2_asr/session.cpp b/src/community_models/confucius4_r2t2/session.cpp similarity index 85% rename from src/community_models/r2t2_asr/session.cpp rename to src/community_models/confucius4_r2t2/session.cpp index 9be0be6a9..a9e2a039d 100644 --- a/src/community_models/r2t2_asr/session.cpp +++ b/src/community_models/confucius4_r2t2/session.cpp @@ -1,10 +1,10 @@ -#include "engine/community_models/r2t2_asr/session.h" +#include "engine/community_models/confucius4_r2t2/session.h" #include "engine/framework/audio/chunking.h" #include "engine/framework/debug/profiler.h" #include "engine/framework/runtime/options.h" #include "engine/framework/runtime/spec_backed_model.h" -#include "engine/community_models/r2t2_asr/text_postprocess.h" +#include "engine/community_models/confucius4_r2t2/text_postprocess.h" #include #include @@ -13,7 +13,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { namespace { using Clock = std::chrono::steady_clock; @@ -44,7 +44,7 @@ void validate_audio_encoder_weight_storage(engine::assets::TensorStorageType sto storage_type == engine::assets::TensorStorageType::F16) { return; } - throw std::runtime_error("r2t2_asr.audio_encoder_weight_type currently supports only native, f32, and f16"); + throw std::runtime_error("confucius4_r2t2.audio_encoder_weight_type currently supports only native, f32, and f16"); } engine::assets::TensorStorageType option_weight_type( @@ -82,15 +82,15 @@ R2T2ASRSession::R2T2ASRSession( : RuntimeSessionBase(options), task_(task), assets_(require_assets(std::move(assets))), - audio_encoder_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.audio_encoder_graph_arena_mb"}, 128ull * 1024ull * 1024ull)), - thinker_prefill_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.thinker_prefill_graph_arena_mb"}, 256ull * 1024ull * 1024ull)), - thinker_decode_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.thinker_decode_graph_arena_mb"}, 256ull * 1024ull * 1024ull)), - thinker_weight_context_bytes_(runtime::parse_size_mb_option(options.options, {"r2t2_asr.thinker_weight_context_mb"}, 64ull * 1024ull * 1024ull)), - audio_encoder_weight_storage_type_(option_weight_type(options, "r2t2_asr.audio_encoder_weight_type", engine::assets::TensorStorageType::Native)), + audio_encoder_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"confucius4_r2t2.audio_encoder_graph_arena_mb"}, 128ull * 1024ull * 1024ull)), + thinker_prefill_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"confucius4_r2t2.thinker_prefill_graph_arena_mb"}, 256ull * 1024ull * 1024ull)), + thinker_decode_graph_arena_bytes_(runtime::parse_size_mb_option(options.options, {"confucius4_r2t2.thinker_decode_graph_arena_mb"}, 256ull * 1024ull * 1024ull)), + thinker_weight_context_bytes_(runtime::parse_size_mb_option(options.options, {"confucius4_r2t2.thinker_weight_context_mb"}, 64ull * 1024ull * 1024ull)), + audio_encoder_weight_storage_type_(option_weight_type(options, "confucius4_r2t2.audio_encoder_weight_type", engine::assets::TensorStorageType::Native)), thinker_weight_storage_type_(option_weight_type( options, - "r2t2_asr.thinker_weight_type", - option_weight_type(options, "r2t2_asr.weight_type", engine::assets::TensorStorageType::Native))), + "confucius4_r2t2.thinker_weight_type", + option_weight_type(options, "confucius4_r2t2.weight_type", engine::assets::TensorStorageType::Native))), tokenizer_(assets_), frontend_(assets_), audio_encoder_(assets_, execution_context(), audio_encoder_graph_arena_bytes_, audio_encoder_weight_storage_type_), @@ -108,47 +108,47 @@ R2T2ASRSession::R2T2ASRSession( throw std::runtime_error("R2T2 ASR supports offline and streaming sessions"); } validate_audio_encoder_weight_storage(audio_encoder_weight_storage_type_); - validate_matmul_weight_storage(thinker_weight_storage_type_, "r2t2_asr.thinker_weight_type"); + validate_matmul_weight_storage(thinker_weight_storage_type_, "confucius4_r2t2.thinker_weight_type"); - if (const auto value = runtime::parse_float_option(options.options, {"r2t2_asr.chunk_size_ms"})) { + if (const auto value = runtime::parse_float_option(options.options, {"confucius4_r2t2.chunk_size_ms"})) { stream_config_.chunk_seconds = static_cast(*value) / 1000.0; } - if (const auto value = runtime::parse_int_option(options.options, {"r2t2_asr.unfixed_chunk_num"})) { + if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.unfixed_chunk_num"})) { stream_config_.unfixed_chunk_num = *value; } - if (const auto value = runtime::parse_int_option(options.options, {"r2t2_asr.unfixed_token_num"})) { + if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.unfixed_token_num"})) { stream_config_.unfixed_token_num = *value; } - if (const auto value = runtime::find_option(options.options, {"r2t2_asr.rollback_punctuation"})) { - stream_config_.rollback_punctuation = runtime::parse_bool_option(*value, "r2t2_asr.rollback_punctuation"); + if (const auto value = runtime::find_option(options.options, {"confucius4_r2t2.rollback_punctuation"})) { + stream_config_.rollback_punctuation = runtime::parse_bool_option(*value, "confucius4_r2t2.rollback_punctuation"); } - if (const auto value = runtime::parse_int_option(options.options, {"r2t2_asr.max_tokens"})) { + if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.max_tokens"})) { stream_config_.max_new_tokens = *value; } if (stream_config_.chunk_seconds <= 0.0) { - throw std::runtime_error("r2t2_asr.chunk_size_ms must be positive"); + throw std::runtime_error("confucius4_r2t2.chunk_size_ms must be positive"); } if (stream_config_.unfixed_chunk_num < 0 || stream_config_.unfixed_token_num < 0) { - throw std::runtime_error("r2t2_asr.unfixed_chunk_num and r2t2_asr.unfixed_token_num must be non-negative"); + throw std::runtime_error("confucius4_r2t2.unfixed_chunk_num and confucius4_r2t2.unfixed_token_num must be non-negative"); } if (stream_config_.max_new_tokens <= 0) { - throw std::runtime_error("r2t2_asr.max_tokens must be positive"); + throw std::runtime_error("confucius4_r2t2.max_tokens must be positive"); } for (const auto & [key, value] : options.options) { (void) value; - if (key.rfind("r2t2_asr.", 0) == 0 && - key != "r2t2_asr.audio_encoder_graph_arena_mb" && - key != "r2t2_asr.thinker_prefill_graph_arena_mb" && - key != "r2t2_asr.thinker_decode_graph_arena_mb" && - key != "r2t2_asr.thinker_weight_context_mb" && - key != "r2t2_asr.audio_encoder_weight_type" && - key != "r2t2_asr.thinker_weight_type" && - key != "r2t2_asr.weight_type" && - key != "r2t2_asr.chunk_size_ms" && - key != "r2t2_asr.unfixed_chunk_num" && - key != "r2t2_asr.unfixed_token_num" && - key != "r2t2_asr.rollback_punctuation" && - key != "r2t2_asr.max_tokens") { + if (key.rfind("confucius4_r2t2.", 0) == 0 && + key != "confucius4_r2t2.audio_encoder_graph_arena_mb" && + key != "confucius4_r2t2.thinker_prefill_graph_arena_mb" && + key != "confucius4_r2t2.thinker_decode_graph_arena_mb" && + key != "confucius4_r2t2.thinker_weight_context_mb" && + key != "confucius4_r2t2.audio_encoder_weight_type" && + key != "confucius4_r2t2.thinker_weight_type" && + key != "confucius4_r2t2.weight_type" && + key != "confucius4_r2t2.chunk_size_ms" && + key != "confucius4_r2t2.unfixed_chunk_num" && + key != "confucius4_r2t2.unfixed_token_num" && + key != "confucius4_r2t2.rollback_punctuation" && + key != "confucius4_r2t2.max_tokens") { throw std::runtime_error("unknown R2T2 ASR session option: " + key); } } @@ -158,7 +158,7 @@ R2T2ASRSession::R2T2ASRSession( R2T2ASRSession::~R2T2ASRSession() = default; std::string R2T2ASRSession::family() const { - return "r2t2_asr"; + return "confucius4_r2t2"; } runtime::VoiceTaskKind R2T2ASRSession::task_kind() const { @@ -215,8 +215,8 @@ R2T2ASRResult R2T2ASRSession::run_single(const R2T2ASRRequest & request) { const auto parsed = parse_asr_output(raw, request.language); result.language = parsed.language.empty() ? request.language : parsed.language; result.text = truncate_at_pipe(parsed.text); - debug::timing_log_scalar("r2t2_asr.single_ms", engine::debug::elapsed_ms(wall_start)); - debug::trace_log_scalar("r2t2_asr.audio_frames", features.frames); + debug::timing_log_scalar("confucius4_r2t2.single_ms", engine::debug::elapsed_ms(wall_start)); + debug::trace_log_scalar("confucius4_r2t2.audio_frames", features.frames); return result; } @@ -376,11 +376,11 @@ R2T2ASRSession::StreamOutcome R2T2ASRSession::decode_stream_chunk(bool final_flu if (!contains_asr_text_tag(raw_decoded_) && force_language_.empty()) { // The model has not emitted the language tag yet: nothing to commit. text_.clear(); - debug::trace_log_scalar("r2t2_asr.stream.final_flush", final_flush ? 1 : 0); - debug::trace_log_scalar("r2t2_asr.stream.chunk_id", chunk_id_); - debug::trace_log_scalar("r2t2_asr.stream.raw_decoded", raw_decoded_); - debug::trace_log_scalar("r2t2_asr.stream.fixed_text", std::string_view{}); - debug::trace_log_scalar("r2t2_asr.stream.text", std::string_view{}); + debug::trace_log_scalar("confucius4_r2t2.stream.final_flush", final_flush ? 1 : 0); + debug::trace_log_scalar("confucius4_r2t2.stream.chunk_id", chunk_id_); + debug::trace_log_scalar("confucius4_r2t2.stream.raw_decoded", raw_decoded_); + debug::trace_log_scalar("confucius4_r2t2.stream.fixed_text", std::string_view{}); + debug::trace_log_scalar("confucius4_r2t2.stream.text", std::string_view{}); return outcome; } @@ -389,11 +389,11 @@ R2T2ASRSession::StreamOutcome R2T2ASRSession::decode_stream_chunk(bool final_flu ++chunk_id_; outcome.text = text_; outcome.fixed_text = fixed_text; - debug::trace_log_scalar("r2t2_asr.stream.final_flush", final_flush ? 1 : 0); - debug::trace_log_scalar("r2t2_asr.stream.chunk_id", chunk_id_); - debug::trace_log_scalar("r2t2_asr.stream.raw_decoded", raw_decoded_); - debug::trace_log_scalar("r2t2_asr.stream.fixed_text", fixed_text); - debug::trace_log_scalar("r2t2_asr.stream.text", text_); + debug::trace_log_scalar("confucius4_r2t2.stream.final_flush", final_flush ? 1 : 0); + debug::trace_log_scalar("confucius4_r2t2.stream.chunk_id", chunk_id_); + debug::trace_log_scalar("confucius4_r2t2.stream.raw_decoded", raw_decoded_); + debug::trace_log_scalar("confucius4_r2t2.stream.fixed_text", fixed_text); + debug::trace_log_scalar("confucius4_r2t2.stream.text", text_); return outcome; } @@ -569,22 +569,22 @@ runtime::TaskResult R2T2ASRSession::finalize() { stream_event_sink_(event); } stream_started_ = false; - debug::timing_log_scalar("r2t2_asr.session.stream.chunks", chunk_id_); - debug::timing_log_scalar("r2t2_asr.session.stream.finalize_ms", engine::debug::elapsed_ms(finalize_start)); + debug::timing_log_scalar("confucius4_r2t2.session.stream.chunks", chunk_id_); + debug::timing_log_scalar("confucius4_r2t2.session.stream.finalize_ms", engine::debug::elapsed_ms(finalize_start)); if (stream_wall_start_ != std::chrono::steady_clock::time_point{}) { - debug::timing_log_scalar("r2t2_asr.session.stream.wall_ms", engine::debug::elapsed_ms(stream_wall_start_)); + debug::timing_log_scalar("confucius4_r2t2.session.stream.wall_ms", engine::debug::elapsed_ms(stream_wall_start_)); debug::timing_log_scalar("session.wall_ms", engine::debug::elapsed_ms(stream_wall_start_)); } return streaming_result_; } -// Loading adapter: r2t2_asr uses the schema-v1 spec-backed loader, so the loader +// Loading adapter: confucius4_r2t2 uses the schema-v1 spec-backed loader, so the loader // wiring stays beside the session it constructs (no per-model loader.{h,cpp}). -std::shared_ptr make_r2t2_asr_loader() { +std::shared_ptr make_confucius4_r2t2_loader() { runtime::SpecBackedVoiceModelConfig config; - config.family = "r2t2_asr"; + config.family = "confucius4_r2t2"; config.load_assets = [](const std::filesystem::path & model_path) { - return load_r2t2_asr_assets(model_path); + return load_confucius4_r2t2_assets(model_path); }; config.create_session = [](const runtime::TaskSpec & task, const runtime::SessionOptions & options, @@ -596,4 +596,4 @@ std::shared_ptr make_r2t2_asr_loader() { return runtime::make_spec_backed_voice_loader(std::move(config)); } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/src/community_models/r2t2_asr/text_postprocess.cpp b/src/community_models/confucius4_r2t2/text_postprocess.cpp similarity index 99% rename from src/community_models/r2t2_asr/text_postprocess.cpp rename to src/community_models/confucius4_r2t2/text_postprocess.cpp index 38d583c40..0b3474f66 100644 --- a/src/community_models/r2t2_asr/text_postprocess.cpp +++ b/src/community_models/confucius4_r2t2/text_postprocess.cpp @@ -1,4 +1,4 @@ -#include "engine/community_models/r2t2_asr/text_postprocess.h" +#include "engine/community_models/confucius4_r2t2/text_postprocess.h" #include #include @@ -7,7 +7,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { namespace { std::vector utf8_to_codepoints(const std::string & text) { @@ -602,4 +602,4 @@ bool ends_with_rollback_punctuation(const std::string & trimmed_text) { return last == ':' || last == ';'; } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/src/community_models/r2t2_asr/thinker.cpp b/src/community_models/confucius4_r2t2/thinker.cpp similarity index 96% rename from src/community_models/r2t2_asr/thinker.cpp rename to src/community_models/confucius4_r2t2/thinker.cpp index 8ab1acecb..6fd5e7764 100644 --- a/src/community_models/r2t2_asr/thinker.cpp +++ b/src/community_models/confucius4_r2t2/thinker.cpp @@ -1,11 +1,11 @@ -#include "engine/community_models/r2t2_asr/thinker.h" +#include "engine/community_models/confucius4_r2t2/thinker.h" #include "engine/framework/runtime/greedy_qwen_decoder.h" #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { namespace { namespace modules = engine::modules; @@ -116,4 +116,4 @@ R2T2ASRGeneratedTokens R2T2ASRThinkerRuntime::generate( return out; } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/src/community_models/r2t2_asr/tokenizer_text.cpp b/src/community_models/confucius4_r2t2/tokenizer_text.cpp similarity index 95% rename from src/community_models/r2t2_asr/tokenizer_text.cpp rename to src/community_models/confucius4_r2t2/tokenizer_text.cpp index bb2aa69dd..50bec16c2 100644 --- a/src/community_models/r2t2_asr/tokenizer_text.cpp +++ b/src/community_models/confucius4_r2t2/tokenizer_text.cpp @@ -1,4 +1,4 @@ -#include "engine/community_models/r2t2_asr/tokenizer_text.h" +#include "engine/community_models/confucius4_r2t2/tokenizer_text.h" #include "engine/framework/tokenizers/llama_bpe.h" @@ -6,7 +6,7 @@ #include #include -namespace engine::community_models::r2t2_asr { +namespace engine::community_models::confucius4_r2t2 { struct R2T2ASRTextTokenizer::Impl { std::shared_ptr tokenizer; @@ -112,4 +112,4 @@ std::string R2T2ASRTextTokenizer::decode(const std::vector & token_ids) return impl_->tokenizer->decode(filtered); } -} // namespace engine::community_models::r2t2_asr +} // namespace engine::community_models::confucius4_r2t2 diff --git a/tests/r2t2_asr/README.md b/tests/confucius4_r2t2/README.md similarity index 75% rename from tests/r2t2_asr/README.md rename to tests/confucius4_r2t2/README.md index 27056e5e1..8884d2c27 100644 --- a/tests/r2t2_asr/README.md +++ b/tests/confucius4_r2t2/README.md @@ -1,6 +1,6 @@ # R2T2 ASR verification -These files verify the `r2t2_asr` family against the macOS MPS reference +These files verify the `confucius4_r2t2` family against the macOS MPS reference implementation in the Confucius4-R2T2 repository (see `docs/community_models/r2t2.md`). ## Files @@ -9,7 +9,7 @@ implementation in the Confucius4-R2T2 repository (see `docs/community_models/r2t |---|---| | `make_golden.py` | Runs the Python reference (`R2T2ASRModel` on MPS) for an audio file and writes offline text plus per-chunk streaming `fixed_text`, `raw_decoded`, and `text` to a golden JSON. | | `compare.py` | Runs `audiocpp_cli` offline and streaming with `--log-file`, parses the per-chunk trace, and diffs everything against a golden. | -| `test_r2t2_asr_transcription.cpp` | Repo-native smoke test: offline + streaming transcripts against the golden for `assets/resources/sample_16k.wav`. Skips with exit code 125 when the model or audio is missing. Also exposes `--encode ` to dump token ids for tokenizer diffing. | +| `test_confucius4_r2t2_transcription.cpp` | Repo-native smoke test: offline + streaming transcripts against the golden for `assets/resources/sample_16k.wav`. Skips with exit code 125 when the model or audio is missing. Also exposes `--encode ` to dump token ids for tokenizer diffing. | | `golden*.json` | Recorded reference outputs. | ## Goldens @@ -31,21 +31,21 @@ Run from the Confucius4-R2T2 checkout (its `uv` environment has torch/MPS): ```bash cd /path/to/Confucius4-R2T2 -PYTHONPATH=. uv run python /path/to/audio.cpp/tests/r2t2_asr/make_golden.py \ +PYTHONPATH=. uv run python /path/to/audio.cpp/tests/confucius4_r2t2/make_golden.py \ --model_path /path/to/audio.cpp/models/Confucius4-R2T2 \ --audio resources/test.wav \ - --out /path/to/audio.cpp/tests/r2t2_asr/golden.json + --out /path/to/audio.cpp/tests/confucius4_r2t2/golden.json ``` ## Comparing ```bash cd /path/to/audio.cpp -python3 tests/r2t2_asr/compare.py \ +python3 tests/confucius4_r2t2/compare.py \ --cli build/macos-metal-release/bin/audiocpp_cli \ --model models/Confucius4-R2T2 \ --audio assets/resources/sample_16k.wav \ - --golden tests/r2t2_asr/golden_sample16k.json \ + --golden tests/confucius4_r2t2/golden_sample16k.json \ --backend metal ``` @@ -58,6 +58,6 @@ stream (the reference WebSocket integrator's rule), every per-chunk committed The same binary can dump the family tokenizer for diffing against Hugging Face: ```bash -build/macos-metal-release/bin/test_r2t2_asr_transcription \ +build/macos-metal-release/bin/test_confucius4_r2t2_transcription \ --encode "language EnglishSome text 22,500" ``` diff --git a/tests/r2t2_asr/compare.py b/tests/confucius4_r2t2/compare.py similarity index 88% rename from tests/r2t2_asr/compare.py rename to tests/confucius4_r2t2/compare.py index 4a496a51b..e3b2015bc 100644 --- a/tests/r2t2_asr/compare.py +++ b/tests/confucius4_r2t2/compare.py @@ -2,11 +2,11 @@ """Compare audiocpp R2T2 output against the macOS MPS golden reference. Usage: - python3 tests/r2t2_asr/compare.py \ + python3 tests/confucius4_r2t2/compare.py \ --cli build/macos-metal-release/bin/audiocpp_cli \ --model models/Confucius4-R2T2 \ --audio \ - --golden tests/r2t2_asr/golden.json \ + --golden tests/confucius4_r2t2/golden.json \ [--backend metal] [--chunk-ms 320] Runs the offline CLI, then the streaming CLI with trace logging enabled, and @@ -42,17 +42,17 @@ def parse_trace(path): continue name = m.group("name") value = m.group("value") - if name == "r2t2_asr.stream.chunk_id": + if name == "confucius4_r2t2.stream.chunk_id": if pending: chunks.append(pending) pending = {"chunk_id": int(value), "final_flush": 0, "fixed_text": None, "text": None, "raw_decoded": None} - elif name == "r2t2_asr.stream.final_flush" and pending: + elif name == "confucius4_r2t2.stream.final_flush" and pending: pending["final_flush"] = int(value) - elif name == "r2t2_asr.stream.fixed_text" and pending: + elif name == "confucius4_r2t2.stream.fixed_text" and pending: pending["fixed_text"] = value - elif name == "r2t2_asr.stream.raw_decoded" and pending: + elif name == "confucius4_r2t2.stream.raw_decoded" and pending: pending["raw_decoded"] = value - elif name == "r2t2_asr.stream.text" and pending: + elif name == "confucius4_r2t2.stream.text" and pending: pending["text"] = value if pending: chunks.append(pending) @@ -105,7 +105,7 @@ def main(): # --- offline ----------------------------------------------------------- offline_cmd = [ - args.cli, "--task", "asr", "--family", "r2t2_asr", "--model", args.model, + args.cli, "--task", "asr", "--family", "confucius4_r2t2", "--model", args.model, "--backend", args.backend, "--audio", args.audio, ] offline = run(offline_cmd) @@ -126,12 +126,12 @@ def main(): with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as tmp: trace_path = tmp.name stream_cmd = [ - args.cli, "--task", "asr", "--mode", "streaming", "--family", "r2t2_asr", + args.cli, "--task", "asr", "--mode", "streaming", "--family", "confucius4_r2t2", "--model", args.model, "--backend", args.backend, "--audio", args.audio, - "--session-option", f"r2t2_asr.chunk_size_ms={chunk_ms}", - "--session-option", f"r2t2_asr.max_tokens={max_new_tokens}", - "--session-option", f"r2t2_asr.unfixed_chunk_num={golden['unfixed_chunk_num']}", - "--session-option", f"r2t2_asr.unfixed_token_num={golden['unfixed_token_num']}", + "--session-option", f"confucius4_r2t2.chunk_size_ms={chunk_ms}", + "--session-option", f"confucius4_r2t2.max_tokens={max_new_tokens}", + "--session-option", f"confucius4_r2t2.unfixed_chunk_num={golden['unfixed_chunk_num']}", + "--session-option", f"confucius4_r2t2.unfixed_token_num={golden['unfixed_token_num']}", "--log-file", trace_path, ] if golden.get("language"): diff --git a/tests/r2t2_asr/golden.json b/tests/confucius4_r2t2/golden.json similarity index 100% rename from tests/r2t2_asr/golden.json rename to tests/confucius4_r2t2/golden.json diff --git a/tests/r2t2_asr/golden_sample16k.json b/tests/confucius4_r2t2/golden_sample16k.json similarity index 100% rename from tests/r2t2_asr/golden_sample16k.json rename to tests/confucius4_r2t2/golden_sample16k.json diff --git a/tests/r2t2_asr/golden_sample16k_bf16.json b/tests/confucius4_r2t2/golden_sample16k_bf16.json similarity index 100% rename from tests/r2t2_asr/golden_sample16k_bf16.json rename to tests/confucius4_r2t2/golden_sample16k_bf16.json diff --git a/tests/r2t2_asr/golden_zh.json b/tests/confucius4_r2t2/golden_zh.json similarity index 100% rename from tests/r2t2_asr/golden_zh.json rename to tests/confucius4_r2t2/golden_zh.json diff --git a/tests/r2t2_asr/make_golden.py b/tests/confucius4_r2t2/make_golden.py similarity index 96% rename from tests/r2t2_asr/make_golden.py rename to tests/confucius4_r2t2/make_golden.py index aa1bb0f76..f66dab49d 100644 --- a/tests/r2t2_asr/make_golden.py +++ b/tests/confucius4_r2t2/make_golden.py @@ -4,9 +4,9 @@ Run from the Confucius4-R2T2 repo (uv environment): cd /Users/david/github/voice/Confucius4-R2T2 - PYTHONPATH=. uv run python /Users/david/github/voice/audio.cpp/tests/r2t2_asr/make_golden.py \ + PYTHONPATH=. uv run python /Users/david/github/voice/audio.cpp/tests/confucius4_r2t2/make_golden.py \ --model_path checkpoints/r2t2 --audio resources/test.wav \ - --out /Users/david/github/voice/audio.cpp/tests/r2t2_asr/golden.json + --out /Users/david/github/voice/audio.cpp/tests/confucius4_r2t2/golden.json Produces offline transcript plus the per-chunk (text, fixed_text) streaming sequence used to verify the C++ LSP streaming port step by step. diff --git a/tests/r2t2_asr/test_r2t2_asr_transcription.cpp b/tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp similarity index 91% rename from tests/r2t2_asr/test_r2t2_asr_transcription.cpp rename to tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp index 6d24878cf..378f78a2d 100644 --- a/tests/r2t2_asr/test_r2t2_asr_transcription.cpp +++ b/tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp @@ -4,8 +4,8 @@ #include "engine/framework/runtime/model.h" #include "engine/framework/runtime/registry.h" #include "engine/framework/runtime/session.h" -#include "engine/community_models/r2t2_asr/assets.h" -#include "engine/community_models/r2t2_asr/tokenizer_text.h" +#include "engine/community_models/confucius4_r2t2/assets.h" +#include "engine/community_models/confucius4_r2t2/tokenizer_text.h" #include #include @@ -27,7 +27,7 @@ constexpr int kExitFail = 1; constexpr int kExitSkip = 125; // Golden output from the macOS MPS reference implementation -// (tests/r2t2_asr/golden_sample16k.json, produced by make_golden.py). +// (tests/confucius4_r2t2/golden_sample16k.json, produced by make_golden.py). const char * kExpectedOffline = "Some call me nature, others call me mother nature. I've been here for over 4.5 billion years, 22,500 times longer than you."; const char * kExpectedStreamFinal = @@ -142,8 +142,8 @@ std::string run_streaming( // Debug aid: print token ids for a text argument so the family tokenizer can be // diffed against the reference Hugging Face tokenizer. int encode_probe(const std::filesystem::path & model_path, const std::string & text) { - auto assets = engine::community_models::r2t2_asr::load_r2t2_asr_assets(model_path, "r2t2_asr"); - engine::community_models::r2t2_asr::R2T2ASRTextTokenizer tokenizer(assets); + auto assets = engine::community_models::confucius4_r2t2::load_confucius4_r2t2_assets(model_path, "confucius4_r2t2"); + engine::community_models::confucius4_r2t2::R2T2ASRTextTokenizer tokenizer(assets); const auto ids = tokenizer.encode(text); std::cout << "count=" << ids.size() << "\nids="; for (size_t i = 0; i < ids.size(); ++i) { @@ -170,7 +170,7 @@ int main(int argc, char ** argv) { if (!model_available || !engine::io::is_existing_file(audio_path)) { std::fprintf( stderr, - "SKIP: test_r2t2_asr_transcription requires model weights at '%s' and audio at '%s'.\n", + "SKIP: test_confucius4_r2t2_transcription requires model weights at '%s' and audio at '%s'.\n", model_path.string().c_str(), audio_path.string().c_str()); return kExitSkip; @@ -180,14 +180,14 @@ int main(int argc, char ** argv) { auto registry = engine::runtime::make_default_registry(); engine::runtime::ModelLoadRequest load_request; load_request.model_path = model_path; - load_request.family_hint = "r2t2_asr"; + load_request.family_hint = "confucius4_r2t2"; auto model = registry.load(load_request); engine::runtime::SessionOptions options; options.backend.type = parse_backend(backend_name); options.backend.threads = 8; - options.options["r2t2_asr.chunk_size_ms"] = std::to_string(kStreamingChunkMs); - options.options["r2t2_asr.max_tokens"] = std::to_string(kStreamingMaxNewTokens); + options.options["confucius4_r2t2.chunk_size_ms"] = std::to_string(kStreamingChunkMs); + options.options["confucius4_r2t2.max_tokens"] = std::to_string(kStreamingMaxNewTokens); const auto audio = read_audio(audio_path); diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json index ad387a4a5..222e7de6b 100644 --- a/webui/configs/model_params.json +++ b/webui/configs/model_params.json @@ -111,12 +111,12 @@ {"name": "audio_chunk_duration_sec", "type": "number", "label": "audio_chunk_duration_sec", "default": 30, "minimum": 0.001, "step": 1} ], - "r2t2_asr": [ - {"name": "chunk_size_ms", "type": "slider", "scope": "session", "session_option": "r2t2_asr.chunk_size_ms", "label": "chunk_size_ms（流式分块毫秒）", "label_en": "chunk_size_ms (streaming chunk, ms)", "default": 320, "minimum": 80, "maximum": 2000, "step": 10, "precision": 0, "info": "80-2000ms：越小延迟越低；320ms 在 Apple Silicon 上延迟与速度较均衡。", "info_en": "80-2000 ms. Lower means lower latency; 320 ms balances latency and speed on Apple Silicon."}, - {"name": "unfixed_chunk_num", "type": "number", "scope": "session", "session_option": "r2t2_asr.unfixed_chunk_num", "label": "unfixed_chunk_num（前 N 块不用稳定前缀）", "label_en": "unfixed_chunk_num (leading chunks without prefix)", "default": 2, "minimum": 0, "maximum": 10, "step": 1, "precision": 0, "info": "开头若干块不使用已识别文本作为前缀提示。", "info_en": "Leading chunks that decode without a stable-prefix prompt."}, - {"name": "unfixed_token_num", "type": "number", "scope": "session", "session_option": "r2t2_asr.unfixed_token_num", "label": "unfixed_token_num（回滚 token 数）", "label_en": "unfixed_token_num (rollback tokens)", "default": 5, "minimum": 0, "maximum": 20, "step": 1, "precision": 0, "info": "作为前缀前从累积文本回滚的 token 数，用于降低边界抖动。", "info_en": "Tokens rolled back from the accumulated text before it is used as the prefix prompt."}, - {"name": "rollback_punctuation", "type": "bool", "scope": "session", "session_option": "r2t2_asr.rollback_punctuation", "label": "rollback_punctuation（句末标点不回滚）", "label_en": "rollback_punctuation (keep trailing punctuation)", "default": false, "info": "输出已以标点结尾时不再回滚 token。", "info_en": "Do not roll back tokens when the output already ends with punctuation."}, - {"name": "max_new_tokens", "type": "number", "scope": "session", "session_option": "r2t2_asr.max_tokens", "label": "max_new_tokens（每分块解码上限）", "label_en": "max_new_tokens (per-chunk decode budget)", "default": 32, "minimum": 1, "maximum": 256, "step": 1, "precision": 0, "info": "每个流式分块的贪婪解码上限（会话级，区别于离线 max_tokens）。", "info_en": "Greedy decode budget per streaming chunk (session-scoped; distinct from offline max_tokens)."} + "confucius4_r2t2": [ + {"name": "chunk_size_ms", "type": "slider", "scope": "session", "session_option": "confucius4_r2t2.chunk_size_ms", "label": "chunk_size_ms（流式分块毫秒）", "label_en": "chunk_size_ms (streaming chunk, ms)", "default": 320, "minimum": 80, "maximum": 2000, "step": 10, "precision": 0, "info": "80-2000ms：越小延迟越低；320ms 在 Apple Silicon 上延迟与速度较均衡。", "info_en": "80-2000 ms. Lower means lower latency; 320 ms balances latency and speed on Apple Silicon."}, + {"name": "unfixed_chunk_num", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.unfixed_chunk_num", "label": "unfixed_chunk_num（前 N 块不用稳定前缀）", "label_en": "unfixed_chunk_num (leading chunks without prefix)", "default": 2, "minimum": 0, "maximum": 10, "step": 1, "precision": 0, "info": "开头若干块不使用已识别文本作为前缀提示。", "info_en": "Leading chunks that decode without a stable-prefix prompt."}, + {"name": "unfixed_token_num", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.unfixed_token_num", "label": "unfixed_token_num（回滚 token 数）", "label_en": "unfixed_token_num (rollback tokens)", "default": 5, "minimum": 0, "maximum": 20, "step": 1, "precision": 0, "info": "作为前缀前从累积文本回滚的 token 数，用于降低边界抖动。", "info_en": "Tokens rolled back from the accumulated text before it is used as the prefix prompt."}, + {"name": "rollback_punctuation", "type": "bool", "scope": "session", "session_option": "confucius4_r2t2.rollback_punctuation", "label": "rollback_punctuation（句末标点不回滚）", "label_en": "rollback_punctuation (keep trailing punctuation)", "default": false, "info": "输出已以标点结尾时不再回滚 token。", "info_en": "Do not roll back tokens when the output already ends with punctuation."}, + {"name": "max_new_tokens", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.max_tokens", "label": "max_new_tokens（每分块解码上限）", "label_en": "max_new_tokens (per-chunk decode budget)", "default": 32, "minimum": 1, "maximum": 256, "step": 1, "precision": 0, "info": "每个流式分块的贪婪解码上限（会话级，区别于离线 max_tokens）。", "info_en": "Greedy decode budget per streaming chunk (session-scoped; distinct from offline max_tokens)."} ], "pocket_tts": [ diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index e176b5aaa..b96996a95 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -119,7 +119,7 @@ { "id": "qwen3-asr", "display_name": "Qwen3-ASR 0.6B (asr)", "family": "qwen3_asr", "path": "models/Qwen3-ASR-0.6B", "task": "asr", "mode": "offline", "download_id": "qwen3_asr_0_6b", "min_vram_gb": 3 }, { "id": "qwen3-asr-1.7b", "display_name": "Qwen3-ASR 1.7B HF (asr)", "family": "qwen3_asr", "path": "models/Qwen3-ASR-1.7B-hf", "task": "asr", "mode": "offline", "download_id": "qwen3_asr_1_7b_hf", "min_vram_gb": 6, "input_hint": "**Qwen3-ASR 1.7B**（HF 原生权重，免转换）：精度高于 0.6B；长音频自动分段转写；8G 卡显存偏紧，长音频建议先短段试跑。", "input_hint_en": "**Qwen3-ASR 1.7B**: native Hugging Face weights with no conversion required. It is more accurate than the 0.6B model and automatically chunks long audio; test short clips first on an 8 GB GPU." }, - { "id": "r2t2-asr", "display_name": "Confucius4-R2T2 (asr, 实时流式)", "display_name_en": "Confucius4-R2T2 (asr, real-time streaming)", "family": "r2t2_asr", "path": "models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", "task": "asr", "mode": "offline", "download_id": "r2t2_asr_q8_0", "min_vram_gb": 5, + { "id": "r2t2-asr", "display_name": "Confucius4-R2T2 (asr, 实时流式)", "display_name_en": "Confucius4-R2T2 (asr, real-time streaming)", "family": "confucius4_r2t2", "path": "models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", "task": "asr", "mode": "offline", "download_id": "confucius4_r2t2_q8_0", "min_vram_gb": 5, "input_hint": "**Confucius4-R2T2**：网易有道实时流式 ASR，Qwen3-ASR-1.7B 微调，LSP 稳定前缀解码；提交文本永不回改，支持 80ms-2s 分块；Q8_0 GGUF（2.3G，自包含单文件），也支持 F16。", "input_hint_en": "**Confucius4-R2T2**: NetEase Youdao real-time streaming ASR, a Qwen3-ASR 1.7B fine-tune with Longest Stable Prefix decoding. Committed text is never revised; 80 ms-2 s chunks; Q8_0 GGUF (2.3 GB, self-contained single file), F16 also available." }, { "id": "niagara-asr-19m", "display_name": "Niagara ASR 19M (asr)", "display_name_en": "Niagara ASR 19M (asr)", "family": "niagara_asr", "path": "models/Niagara-ASR-GGUF/niagara-19m-batch.en-f32.gguf", "task": "asr", "mode": "offline", "download_id": "niagara_19m_f32", "min_vram_gb": 1, "input_hint": "**Niagara ASR 19M**：ABR 英语离线 ASR，F32 GGUF 权重。", "input_hint_en": "**Niagara ASR 19M**: ABR English offline ASR with F32 GGUF weights." }, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index e10d8f71f..2f173ad9a 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,19 +31,19 @@
diff --git a/webui/native/src/lib/catalog.ts b/webui/native/src/lib/catalog.ts index 2adf0c7a5..b00e471d2 100644 --- a/webui/native/src/lib/catalog.ts +++ b/webui/native/src/lib/catalog.ts @@ -43,7 +43,7 @@ const exposeAllGgufPackageFamilies = new Set([ 'canary_asr', 'cohere_asr', 'moss_transcribe_diarize', - 'r2t2_asr', + 'confucius4_r2t2', 'audiosr', 'controlfoley', 'breeze_tts', diff --git a/webui/native/src/routes/+page.svelte b/webui/native/src/routes/+page.svelte index ce1b4acf3..dad7fd588 100644 --- a/webui/native/src/routes/+page.svelte +++ b/webui/native/src/routes/+page.svelte @@ -318,7 +318,7 @@ cohere_asr: ['en', 'fr', 'de', 'es', 'it', 'pt', 'nl', 'pl', 'el', 'ar', 'ja', 'zh', 'vi', 'ko'], // Confucius4-R2T2 takes canonical language names (the engine normalizes // case); 'Auto' leaves language detection on. - r2t2_asr: ['Auto', 'Chinese', 'English', 'Cantonese', 'Japanese', 'Korean', 'Arabic', 'German', 'French', 'Spanish', 'Portuguese', 'Indonesian', 'Italian', 'Russian', 'Thai', 'Vietnamese', 'Turkish', 'Hindi', 'Malay', 'Dutch', 'Swedish', 'Danish', 'Finnish', 'Polish', 'Czech', 'Filipino', 'Persian', 'Greek', 'Romanian', 'Hungarian', 'Macedonian'] + confucius4_r2t2: ['Auto', 'Chinese', 'English', 'Cantonese', 'Japanese', 'Korean', 'Arabic', 'German', 'French', 'Spanish', 'Portuguese', 'Indonesian', 'Italian', 'Russian', 'Thai', 'Vietnamese', 'Turkish', 'Hindi', 'Malay', 'Dutch', 'Swedish', 'Danish', 'Finnish', 'Polish', 'Czech', 'Filipino', 'Persian', 'Greek', 'Romanian', 'Hungarian', 'Macedonian'] }; function pathVariantLabel(path: string) { @@ -479,7 +479,7 @@ !['apollo', 'universr'].includes(selected?.family) && !replacesGenericControls.text; $: supportsLiveAsr = selected?.task === 'asr' && - ['voxtral_realtime', 'nemotron_asr', 'higgs_audio_stt', 'sense_asr', 'vibevoice_asr_streaming', 'r2t2_asr'].includes(selected?.family); + ['voxtral_realtime', 'nemotron_asr', 'higgs_audio_stt', 'sense_asr', 'vibevoice_asr_streaming', 'confucius4_r2t2'].includes(selected?.family); $: modelInventoryLoading = server === null || (Boolean(server.ui_management) && Object.keys(packageSizes).length === 0 && packageSizeState !== 'failed'); $: selectableModelIds = new Set(activeCatalog.filter((entry) => { @@ -1050,7 +1050,7 @@ !(hidesDurationSec && spec.name === 'duration_sec')); advancedValues = Object.fromEntries(byId.map((spec) => [spec.name, spec.default ?? ''])); if (selected?.family in asrTokenDefaults) asrMaxTokens = asrTokenDefaults[selected.family]; - if (selected?.family === 'r2t2_asr') language = 'Auto'; + if (selected?.family === 'confucius4_r2t2') language = 'Auto'; else if (selected?.family in asrLanguages) language = 'en'; if (selected?.family === 'minimax_h3') { duration = 15; From 8747e0138ad3d3738f815f1d2e3fa59e23212c71 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 11:29:47 +0900 Subject: [PATCH 4/9] docs(r2t2): track streaming graph reuse follow-up --- docs/community_models/r2t2.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index 924d03a2e..381a2610d 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -170,6 +170,25 @@ positions, a single unsegmented stream is limited to roughly 110 s of accumulated audio; segment longer streams (as the reference server does) or add the rolling-window variant. +### Follow-up: reuse streaming graphs + +Streaming currently rebuilds all major inference graphs as each growing +chunk changes their shapes or capacity requirements: + +* The audio encoder graph matches the exact accumulated feature-frame count. +* The thinker prefill graph matches the prompt length and audio-token count. +* The thinker decode graph is replaced when the growing prompt plus decode + budget exceeds its allocated KV-cache capacity. + +Weights remain loaded, but these graph allocations and builds add overhead to +successive chunks; reusing the session alone does not eliminate that cost. +This is follow-up performance work, not a merge blocker. Investigate padded or +bucketed encoder/prefill shapes and a reusable decode graph with reserved KV +capacity. Measure graph-build time, per-chunk latency, and peak memory over +long utterances, and rerun transcript/delta parity checks before adopting the +optimization. The current port still recomputes the accumulated audio and +prompt on each chunk; incremental encoder/prefill caching is a separate task. + ## Server usage `server.local.json` declares `r2t2-asr` (offline) and `r2t2-asr-stream` From fe9e59af2ac975ed088c869c2939b3ab66ce0c19 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 11:48:14 +0900 Subject: [PATCH 5/9] docs(r2t2): keep release-specific GGUF details on the model card --- docs/community_models/r2t2.md | 25 +++++++++---------------- tests/confucius4_r2t2/README.md | 8 +++++--- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index 381a2610d..1138666a2 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -225,10 +225,15 @@ Verified conversions are published at (a quantized derivative work, distributed under the upstream NetEase Youdao model license — see the repository's `LICENSE`, `LICENSE_zh`, and `NOTICE`): -| Package id | File | Quantization | Size | -|---|---|---|---:| -| `confucius4_r2t2_q8_0` (default) | `r2t2-q8_0.gguf` | Q8_0 | 2.31 GiB | -| `confucius4_r2t2_f16` | `r2t2-f16.gguf` | F16 | 3.81 GiB | +| Package id | File | Quantization | +|---|---|---| +| `confucius4_r2t2_q8_0` (default) | `r2t2-q8_0.gguf` | Q8_0 | +| `confucius4_r2t2_f16` | `r2t2-f16.gguf` | F16 | + +For current file sizes, SHA-256 checksums, runtime compatibility, and published +checkpoint validation, see the [Hugging Face model card](https://huggingface.co/davidxifeng/Confucius4-R2T2-gguf). +Release-specific artifact details are maintained there rather than duplicated +in this document. ```bash python3 tools/model_manager_v2.py install confucius4_r2t2_q8_0 # or confucius4_r2t2_f16 @@ -273,18 +278,6 @@ The published checkpoint ties the LM head to the token embedding and therefore contains no `lm_head.weight`; the family detects that and reuses the embedding, so conversions need no special flags. -### Verified GGUF results - -| Checkpoint | Size | Offline | Committed stream | Final transcript | Per-chunk | -|---|---:|---|---|---|---| -| `model.safetensors` (bf16) | 3.80 GiB | exact | exact | exact | 21/21 | -| `--type f16` GGUF | 3.81 GiB | exact | exact | exact | 21/21 | -| `--type q8_0` GGUF | 2.31 GiB | exact | exact | exact | 21/21 | -| `--type q4_k` / `q4_0` | — | rejected at load with a clear error | | | | - -No degradation is measurable at Q8_0 on the verification clip; it is the -recommended distribution format and the default package for this family. - ## Verification The port is verified against the macOS MPS reference with golden traces: diff --git a/tests/confucius4_r2t2/README.md b/tests/confucius4_r2t2/README.md index 8884d2c27..25cb28ffb 100644 --- a/tests/confucius4_r2t2/README.md +++ b/tests/confucius4_r2t2/README.md @@ -9,7 +9,7 @@ implementation in the Confucius4-R2T2 repository (see `docs/community_models/r2t |---|---| | `make_golden.py` | Runs the Python reference (`R2T2ASRModel` on MPS) for an audio file and writes offline text plus per-chunk streaming `fixed_text`, `raw_decoded`, and `text` to a golden JSON. | | `compare.py` | Runs `audiocpp_cli` offline and streaming with `--log-file`, parses the per-chunk trace, and diffs everything against a golden. | -| `test_confucius4_r2t2_transcription.cpp` | Repo-native smoke test: offline + streaming transcripts against the golden for `assets/resources/sample_16k.wav`. Skips with exit code 125 when the model or audio is missing. Also exposes `--encode ` to dump token ids for tokenizer diffing. | +| `test_confucius4_r2t2_transcription.cpp` | Repo-native smoke test: offline + final streaming transcripts against the golden, plus a check that Auto-language deltas form a nonempty prefix of the expected transcript for `assets/resources/sample_16k.wav`. Skips with exit code 125 when the model or audio is missing. Also exposes `--encode ` to dump token ids for tokenizer diffing. | | `golden*.json` | Recorded reference outputs. | ## Goldens @@ -50,8 +50,10 @@ python3 tests/confucius4_r2t2/compare.py \ ``` The comparison checks four things: the offline transcript, the committed delta -stream (the reference WebSocket integrator's rule), every per-chunk committed -`fixed_text`, and the final streaming transcript. See the results table in +stream, every per-chunk committed `fixed_text`, and the final streaming +transcript. Metadata-only rollback prefixes are filtered from the unmodified +reference goldens before comparing committed text; `language` fragments must +not appear in the emitted deltas. See the results table in `docs/community_models/r2t2.md` for what is exact and the two documented internal (non-observable) differences on the English clip. From df883c101e7465d33f2515fbdde81eafa3cebbc6 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 12:43:38 +0900 Subject: [PATCH 6/9] perf(r2t2): reuse streaming encoder, prefill and decode graphs Streaming previously rebuilt every major graph per chunk: the encoder matched the exact accumulated frame count, the thinker prefill matched the prompt length, and the decode graph was replaced whenever the growing prompt outgrew its KV capacity. - Audio encoder gains an opt-in capacity-bucketed graph (1-2 chunks exact, then 4-chunk steps) with a per-run attention-mask refill for the valid token prefix; the offline path keeps exact per-frame graphs. - GreedyQwenDecoderRuntime::generate routes reuse_graphs requests through one reserved block-prefill graph (64-token blocks) and one decode graph with KV capacity grown in 128-token buckets, plus a fixed-width prompt embedding lookup graph; new prompts clear KV on device. - Add block-prefill graph build timing traces. On the M3 Metal 44-chunk golden clip this cuts graph builds from 44/44/44 to 8/13/3, graph-build time from ~1.06 s to ~0.10 s, stream wall time from 25.25 s to 23.67 s, and peak footprint from 7.32 GB to 7.14 GB with identical transcripts. test_confucius4_r2t2_graph_reuse pins the semantics: bit-exact encoder output when a bucket adds no padded tokens, a 2e-2 relative-RMSE noise budget plus run-to-run determinism when it does (ggml reduction order changes with sequence length), and decoder token parity across bucket growth and shrink. --- CMakeLists.txt | 3 + docs/community_models/r2t2.md | 51 +++++--- .../confucius4_r2t2/audio_encoder.h | 2 +- .../community_models/confucius4_r2t2/types.h | 1 + .../framework/runtime/greedy_qwen_decoder.h | 4 +- .../confucius4_r2t2/audio_encoder.cpp | 66 +++++++--- .../confucius4_r2t2/session.cpp | 3 +- .../confucius4_r2t2/thinker.cpp | 2 +- .../qwen_causal_decode_runtime.cpp | 3 + src/framework/runtime/greedy_qwen_decoder.cpp | 119 +++++++++++++++++- tests/confucius4_r2t2/test_graph_reuse.cpp | 117 +++++++++++++++++ 11 files changed, 332 insertions(+), 39 deletions(-) create mode 100644 tests/confucius4_r2t2/test_graph_reuse.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fef666dd2..a114aada1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2988,6 +2988,9 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST endif() if (confucius4_r2t2 IN_LIST AUDIOCPP_LINKED_MODELS) + add_executable(test_confucius4_r2t2_graph_reuse tests/confucius4_r2t2/test_graph_reuse.cpp) + target_compile_definitions(test_confucius4_r2t2_graph_reuse PRIVATE ENGINE_REPO_ROOT="${CMAKE_CURRENT_SOURCE_DIR}") + target_link_libraries(test_confucius4_r2t2_graph_reuse PRIVATE engine_runtime ggml) add_executable(test_confucius4_r2t2_transcription tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp ) diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index 1138666a2..c1d8ec723 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -170,24 +170,39 @@ positions, a single unsegmented stream is limited to roughly 110 s of accumulated audio; segment longer streams (as the reference server does) or add the rolling-window variant. -### Follow-up: reuse streaming graphs - -Streaming currently rebuilds all major inference graphs as each growing -chunk changes their shapes or capacity requirements: - -* The audio encoder graph matches the exact accumulated feature-frame count. -* The thinker prefill graph matches the prompt length and audio-token count. -* The thinker decode graph is replaced when the growing prompt plus decode - budget exceeds its allocated KV-cache capacity. - -Weights remain loaded, but these graph allocations and builds add overhead to -successive chunks; reusing the session alone does not eliminate that cost. -This is follow-up performance work, not a merge blocker. Investigate padded or -bucketed encoder/prefill shapes and a reusable decode graph with reserved KV -capacity. Measure graph-build time, per-chunk latency, and peak memory over -long utterances, and rerun transcript/delta parity checks before adopting the -optimization. The current port still recomputes the accumulated audio and -prompt on each chunk; incremental encoder/prefill caching is a separate task. +### Streaming graph reuse + +Streaming no longer rebuilds the major inference graphs on every chunk. +Opt-in via `encode(features, reuse_graph)` and +`R2T2ASRGenerationOptions::reuse_graphs` (the streaming session enables both; +the offline path keeps exact per-frame graphs): + +* The audio encoder builds one graph sized to a capacity bucket (one to two + chunks exactly, then four-chunk steps) and refills the attention mask for + the valid token prefix on each run, so a growing stream reuses the graph + until the next bucket boundary. +* The thinker prefill runs through the shared Qwen chunked prefill runtime + (64-token blocks, one reserved graph) and the decode graph keeps a KV-cache + capacity grown in 128-token buckets up to `max_position_embeddings`; new + prompts clear KV on device. +* Prompt token embeddings reuse a fixed-width lookup graph instead of a + per-request build. + +Measured on an M3 (Metal) over the 40 s golden clip (44 chunks): graph builds +drop from 44 encoder + 44 prefill + 44 decode to 8 + 13 + 3, total graph-build +time falls from about 1.06 s to 0.10 s, streaming wall time from 25.25 s to +23.67 s, and peak memory footprint from 7.32 GB to 7.14 GB. Transcripts are +identical with and without reuse. + +`test_confucius4_r2t2_graph_reuse` guards the semantics: encoder output is +bit-exact whenever the capacity bucket adds no padded tokens; padded runs are +held to a 2e-2 relative-RMSE noise budget (ggml reduction order changes with +sequence length, so bit equality is impossible once padding exists) and must +be deterministic across runs; decoder output must match the exact path token +for token across bucket growth and shrink. + +The port still recomputes the accumulated audio and prompt on each chunk; +incremental encoder/prefill caching is a separate task. ## Server usage diff --git a/include/engine/community_models/confucius4_r2t2/audio_encoder.h b/include/engine/community_models/confucius4_r2t2/audio_encoder.h index e125cb98e..5b1b2cda7 100644 --- a/include/engine/community_models/confucius4_r2t2/audio_encoder.h +++ b/include/engine/community_models/confucius4_r2t2/audio_encoder.h @@ -22,7 +22,7 @@ class R2T2ASRAudioEncoderRuntime { assets::TensorStorageType weight_storage_type); ~R2T2ASRAudioEncoderRuntime(); - R2T2ASRAudioEmbeddings encode(const R2T2ASRAudioFeatures & features); + R2T2ASRAudioEmbeddings encode(const R2T2ASRAudioFeatures & features, bool reuse_graph = false); private: std::shared_ptr assets_; diff --git a/include/engine/community_models/confucius4_r2t2/types.h b/include/engine/community_models/confucius4_r2t2/types.h index 6ed71eac7..3b7e8b1f0 100644 --- a/include/engine/community_models/confucius4_r2t2/types.h +++ b/include/engine/community_models/confucius4_r2t2/types.h @@ -11,6 +11,7 @@ namespace engine::community_models::confucius4_r2t2 { struct R2T2ASRGenerationOptions { int64_t max_new_tokens = 512; + bool reuse_graphs = false; bool return_timestamps = false; bool clamp_timestamps_to_audio = false; }; diff --git a/include/engine/framework/runtime/greedy_qwen_decoder.h b/include/engine/framework/runtime/greedy_qwen_decoder.h index d54b6b1d3..162532aee 100644 --- a/include/engine/framework/runtime/greedy_qwen_decoder.h +++ b/include/engine/framework/runtime/greedy_qwen_decoder.h @@ -62,7 +62,9 @@ class GreedyQwenDecoderRuntime { GreedyQwenDecoderRuntime(const GreedyQwenDecoderRuntime &) = delete; GreedyQwenDecoderRuntime & operator=(const GreedyQwenDecoderRuntime &) = delete; - std::vector generate(const Prompt & prompt, int64_t max_new_tokens); + // Opt-in bounded-block prefill and capacity-bucketed decode for repeated + // growing prompts. Each call still recomputes the full prompt. + std::vector generate(const Prompt & prompt, int64_t max_new_tokens, bool reuse_graphs = false); private: struct Impl; diff --git a/src/community_models/confucius4_r2t2/audio_encoder.cpp b/src/community_models/confucius4_r2t2/audio_encoder.cpp index 787913b01..6670cef89 100644 --- a/src/community_models/confucius4_r2t2/audio_encoder.cpp +++ b/src/community_models/confucius4_r2t2/audio_encoder.cpp @@ -353,13 +353,15 @@ class R2T2ASRAudioEncoderGraph { std::shared_ptr weights, core::ExecutionContext & execution, size_t graph_arena_bytes, - int64_t frames) + int64_t frames, + bool reusable) : assets_(std::move(assets)), weights_(std::move(weights)), backend_(execution.backend()), backend_type_(execution.backend_type()), compute_threads_(std::max(1, execution.config().threads)), - frames_(frames) { + frames_(frames), + reusable_(reusable) { if (assets_ == nullptr || weights_ == nullptr) { throw std::runtime_error("R2T2 ASR audio encoder graph requires assets and weights"); } @@ -382,7 +384,7 @@ class R2T2ASRAudioEncoderGraph { output_tokens_ = sum_values(chunk_token_lengths_); const int64_t max_chunk_tokens = max_value(chunk_token_lengths_); attention_window_tokens_ = max_chunk_tokens * (config.n_window_infer / chunk_frame_limit_); - if (output_tokens_ > config.max_source_positions) { + if (!reusable_ && output_tokens_ > config.max_source_positions) { throw std::runtime_error("R2T2 ASR audio encoder token count exceeds max_source_positions"); } attention_window_lengths_ = audio_attention_window_lengths(output_tokens_, attention_window_tokens_); @@ -493,25 +495,31 @@ class R2T2ASRAudioEncoderGraph { engine::core::release_backend_graph_resources(backend_, graph_, true); } - bool matches(const R2T2ASRAudioEncoderWeights & weights, int64_t frames, ggml_backend_t backend, int threads) const { - return weights_.get() == &weights && frames_ == frames && backend_ == backend && compute_threads_ == std::max(1, threads); + bool matches(const R2T2ASRAudioEncoderWeights & weights, int64_t frames, ggml_backend_t backend, int threads, bool reusable) const { + // Short inputs keep their exact convolution width. Once a full chunk + // exists, the reference already pads the final chunk to this width. + const bool shape_matches = reusable_ && reusable + ? frames >= chunk_frame_limit_ && frames <= frames_ + : !reusable_ && !reusable && frames_ == frames; + return weights_.get() == &weights && shape_matches && backend_ == backend && compute_threads_ == std::max(1, threads); } R2T2ASRAudioEmbeddings run(const R2T2ASRAudioFeatures & features) { const auto & config = assets_->config.audio_encoder; - if (features.mel_bins != config.num_mel_bins || features.frames != frames_) { + if (features.mel_bins != config.num_mel_bins || (reusable_ ? features.frames > frames_ || features.frames < chunk_frame_limit_ : features.frames != frames_)) { throw std::runtime_error("R2T2 ASR audio encoder feature shape mismatch"); } - if (static_cast(features.values.size()) != config.num_mel_bins * frames_) { + if (static_cast(features.values.size()) != config.num_mel_bins * features.frames) { throw std::runtime_error("R2T2 ASR audio encoder feature value count mismatch"); } std::vector padded_features(static_cast(chunk_count_ * config.num_mel_bins * chunk_frames_), 0.0F); + const auto lengths = audio_chunk_lengths(features.frames, chunk_frame_limit_); int64_t source_frame = 0; - for (int64_t chunk = 0; chunk < chunk_count_; ++chunk) { - const int64_t chunk_length = chunk_lengths_[static_cast(chunk)]; + for (int64_t chunk = 0; chunk < static_cast(lengths.size()); ++chunk) { + const int64_t chunk_length = lengths[static_cast(chunk)]; for (int64_t mel = 0; mel < config.num_mel_bins; ++mel) { const size_t dst = static_cast((chunk * config.num_mel_bins + mel) * chunk_frames_); - const size_t src = static_cast(mel * frames_ + source_frame); + const size_t src = static_cast(mel * features.frames + source_frame); std::copy_n( features.values.begin() + static_cast(src), static_cast(chunk_length), @@ -519,6 +527,18 @@ class R2T2ASRAudioEncoderGraph { } source_frame += chunk_length; } + if (reusable_) { + // Real queries see exactly the reference attention window. Padded + // queries attend only themselves, avoiding all-masked softmax rows. + std::fill(attention_mask_values_.begin(), attention_mask_values_.end(), -INFINITY); + const int64_t valid = features.encoder_tokens; + for (int64_t row = 0; row < output_tokens_; ++row) { + const int64_t begin = row < valid ? row / attention_window_tokens_ * attention_window_tokens_ : row; + const int64_t end = row < valid ? std::min(valid, begin + attention_window_tokens_) : row + 1; + std::fill(attention_mask_values_.begin() + row * output_tokens_ + begin, + attention_mask_values_.begin() + row * output_tokens_ + end, 0.0F); + } + } auto timing_start = Clock::now(); ggml_backend_tensor_set(input_, padded_features.data(), 0, padded_features.size() * sizeof(float)); ggml_backend_tensor_set(attention_mask_, attention_mask_values_.data(), 0, attention_mask_values_.size() * sizeof(float)); @@ -532,7 +552,7 @@ class R2T2ASRAudioEncoderGraph { throw std::runtime_error("R2T2 ASR audio encoder graph compute failed"); } R2T2ASRAudioEmbeddings out; - out.tokens = output_tokens_; + out.tokens = features.encoder_tokens; out.hidden_size = output_dim_; out.values.resize(static_cast(out.tokens * out.hidden_size)); timing_start = Clock::now(); @@ -548,6 +568,7 @@ class R2T2ASRAudioEncoderGraph { core::BackendType backend_type_ = core::BackendType::Cpu; int compute_threads_ = 1; int64_t frames_ = 0; + bool reusable_ = false; int64_t chunk_frame_limit_ = 0; int64_t chunk_frames_ = 0; int64_t chunk_count_ = 0; @@ -585,26 +606,41 @@ R2T2ASRAudioEncoderRuntime::R2T2ASRAudioEncoderRuntime( R2T2ASRAudioEncoderRuntime::~R2T2ASRAudioEncoderRuntime() = default; -R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeatures & features) { +R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeatures & features, bool reuse_graph) { if (execution_ == nullptr) { throw std::runtime_error("R2T2 ASR audio encoder execution context is null"); } if (features.encoder_tokens != confucius4_r2t2_audio_encoder_token_count(features.frames)) { throw std::runtime_error("R2T2 ASR audio encoder token count mismatch"); } + const auto & config = assets_->config.audio_encoder; + if (features.encoder_tokens > config.max_source_positions) { + throw std::runtime_error("R2T2 ASR audio encoder token count exceeds max_source_positions"); + } + const int64_t chunk_frames = config.n_window * 2; + const bool reusable = reuse_graph && features.frames >= chunk_frames; const int threads = std::max(1, execution_->config().threads); - if (graph_ == nullptr || !graph_->matches(*weights_, features.frames, execution_->backend(), threads)) { + if (graph_ == nullptr || !graph_->matches(*weights_, features.frames, execution_->backend(), threads, reusable)) { + int64_t capacity = features.frames; + if (reusable) { + const int64_t chunks = (features.frames + chunk_frames - 1) / chunk_frames; + // At most one graph is retained. Grow in four-chunk buckets after + // the first two chunks, bounding padding overhead and memory. + capacity = (chunks <= 2 ? chunks : (chunks + 3) / 4 * 4) * chunk_frames; + } graph_.reset(); graph_ = std::make_unique( assets_, weights_, *execution_, graph_arena_bytes_, - features.frames); + capacity, + reusable); } else { debug::timing_log_scalar("confucius4_r2t2.audio_encoder.graph.build_ms", 0.0); - debug::trace_log_scalar("confucius4_r2t2.audio_encoder.frames", features.frames); + } + debug::trace_log_scalar("confucius4_r2t2.audio_encoder.input_frames", features.frames); auto out = graph_->run(features); if (out.tokens != features.encoder_tokens) { throw std::runtime_error("R2T2 ASR audio encoder output token count mismatch"); diff --git a/src/community_models/confucius4_r2t2/session.cpp b/src/community_models/confucius4_r2t2/session.cpp index a9e2a039d..68853c0a0 100644 --- a/src/community_models/confucius4_r2t2/session.cpp +++ b/src/community_models/confucius4_r2t2/session.cpp @@ -274,6 +274,7 @@ std::string R2T2ASRSession::generate_text( const R2T2ASRAudioEmbeddings & embeddings) { R2T2ASRGenerationOptions options; options.max_new_tokens = stream_config_.max_new_tokens; + options.reuse_graphs = true; const auto tokens = thinker_.generate(prompt, embeddings, options); return tokenizer_.decode(tokens.token_ids); } @@ -329,7 +330,7 @@ R2T2ASRSession::StreamOutcome R2T2ASRSession::decode_stream_chunk(bool final_flu accum.samples = audio_accum_; const auto features = frontend_.extract(accum); const auto prompt = tokenizer_.build_raw_audio_prompt(prompt_raw_ + prefix, features.encoder_tokens); - const auto embeddings = audio_encoder_.encode(features); + const auto embeddings = audio_encoder_.encode(features, /*reuse_graph=*/true); std::string generated = generate_text(prompt, embeddings); generated = normalize_punct_by_context(generated); generated = sanitize_utf8_lossy(generated); diff --git a/src/community_models/confucius4_r2t2/thinker.cpp b/src/community_models/confucius4_r2t2/thinker.cpp index 6fd5e7764..e6c9933b7 100644 --- a/src/community_models/confucius4_r2t2/thinker.cpp +++ b/src/community_models/confucius4_r2t2/thinker.cpp @@ -112,7 +112,7 @@ R2T2ASRGeneratedTokens R2T2ASRThinkerRuntime::generate( decoder_prompt.injection.positions = prompt.audio_token_positions; R2T2ASRGeneratedTokens out; - out.token_ids = impl_->runtime.generate(decoder_prompt, options.max_new_tokens); + out.token_ids = impl_->runtime.generate(decoder_prompt, options.max_new_tokens, options.reuse_graphs); return out; } diff --git a/src/framework/modules/transformers/qwen_causal_decode_runtime.cpp b/src/framework/modules/transformers/qwen_causal_decode_runtime.cpp index e265f313a..37637bdb4 100644 --- a/src/framework/modules/transformers/qwen_causal_decode_runtime.cpp +++ b/src/framework/modules/transformers/qwen_causal_decode_runtime.cpp @@ -1763,6 +1763,7 @@ class QwenCausalDecodeRuntime::Impl { } void build_block_graph(int64_t chunk) { + const auto build_start = Clock::now(); block_ctx_.reset(ggml_init({config_.prefill_graph_arena_bytes, nullptr, true})); if (!block_ctx_) { throw std::runtime_error("Qwen chunked prefill context allocation failed"); } core::ModuleBuildContext ctx{block_ctx_.get(), config_.trace_name.c_str(), backend_type_}; @@ -1811,6 +1812,8 @@ class QwenCausalDecodeRuntime::Impl { throw std::runtime_error("Qwen chunked prefill graph allocation failed"); } block_steps_ = chunk; + debug::timing_log_scalar(config_.trace_name + ".block_prefill.graph.build_ms", + engine::debug::elapsed_ms(build_start, Clock::now())); } void release_block_graph() { diff --git a/src/framework/runtime/greedy_qwen_decoder.cpp b/src/framework/runtime/greedy_qwen_decoder.cpp index e9185c23c..bcbb5f507 100644 --- a/src/framework/runtime/greedy_qwen_decoder.cpp +++ b/src/framework/runtime/greedy_qwen_decoder.cpp @@ -9,6 +9,7 @@ #include "engine/framework/modules/structural_modules.h" #include "engine/framework/runtime/errors.h" #include "engine/framework/runtime/kv_cache.h" +#include "engine/framework/modules/transformers/qwen_causal_decode_runtime.h" #include "engine/framework/sampling/decode_modules.h" #include @@ -297,6 +298,59 @@ class ThinkerWeightsRuntime { std::shared_ptr weights_; }; +// A small fixed-width lookup graph also works for quantized embedding weights. +// Padding is read back only for the last block, then discarded before injection. +class PromptEmbeddingGraph { +public: + static constexpr int64_t kSteps = 64; + explicit PromptEmbeddingGraph(std::shared_ptr runtime) + : runtime_(std::move(runtime)) { + ctx_.reset(ggml_init({1024 * 1024, nullptr, true})); + if (!ctx_) { throw std::runtime_error("failed to initialize prompt embedding graph"); } + core::ModuleBuildContext ctx{ctx_.get(), "greedy_qwen_decoder.embedding", runtime_->backend_type()}; + ids_ = ggml_new_tensor_1d(ctx.ggml, GGML_TYPE_I32, kSteps); + auto ids = core::wrap_tensor(ids_, core::TensorShape::from_dims({kSteps}), GGML_TYPE_I32); + const auto & spec = runtime_->spec(); + output_ = modules::EmbeddingModule({spec.vocab_size, spec.decoder.stack.hidden_size}) + .build(ctx, ids, runtime_->weights().token_embedding).tensor; + ggml_set_output(output_); + graph_ = ggml_new_graph_custom(ctx.ggml, 64, false); + ggml_build_forward_expand(graph_, output_); + allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(runtime_->backend()))); + if (!allocator_ || !ggml_gallocr_alloc_graph(allocator_.get(), graph_)) { + throw std::runtime_error("failed to allocate prompt embedding graph"); + } + } + ~PromptEmbeddingGraph() { + core::release_backend_graph_resources(runtime_->backend(), graph_, true); + } + std::vector run(const std::vector & ids) { + const int64_t width = runtime_->spec().decoder.stack.hidden_size; + std::vector result(ids.size() * static_cast(width)); + std::vector block(kSteps, 0); + core::set_backend_threads(runtime_->backend(), runtime_->threads()); + for (size_t offset = 0; offset < ids.size(); offset += kSteps) { + const size_t count = std::min(kSteps, ids.size() - offset); + std::fill(block.begin(), block.end(), 0); + std::copy_n(ids.begin() + offset, count, block.begin()); + ggml_backend_tensor_set(ids_, block.data(), 0, block.size() * sizeof(int32_t)); + if (core::compute_backend_graph(runtime_->backend(), graph_) != GGML_STATUS_SUCCESS) { + throw std::runtime_error("prompt embedding graph compute failed"); + } + ggml_backend_synchronize(runtime_->backend()); + ggml_backend_tensor_get(output_, result.data() + offset * width, 0, count * width * sizeof(float)); + } + return result; + } +private: + std::shared_ptr runtime_; + std::unique_ptr ctx_; + ggml_tensor * ids_ = nullptr; + ggml_tensor * output_ = nullptr; + ggml_cgraph * graph_ = nullptr; + std::unique_ptr, GgmlGallocrDeleter> allocator_; +}; + class PrefillGraph { public: PrefillGraph( @@ -605,13 +659,66 @@ struct GreedyQwenDecoderRuntime::Impl { weight_context_bytes, storage_type)), prefill_graph_arena_bytes(prefill_graph_arena_bytes), - decode_graph_arena_bytes(decode_graph_arena_bytes) {} + decode_graph_arena_bytes(decode_graph_arena_bytes), + execution(&execution) {} std::shared_ptr weights; size_t prefill_graph_arena_bytes = 0; size_t decode_graph_arena_bytes = 0; std::unique_ptr prefill_graph; std::unique_ptr decode_graph; + core::ExecutionContext * execution; + std::unique_ptr embedding_graph; + std::unique_ptr reusable_decoder; + + std::vector generate_reusing_graphs(const GreedyQwenDecoderRuntime::Prompt & prompt, int64_t max_new_tokens) { + const auto & spec = weights->spec(); + if (!reusable_decoder) { + modules::QwenCausalDecodeRuntimeConfig config; + config.trace_name = "greedy_qwen_decoder.reusable"; + config.decoder = spec.decoder; + config.prefill_graph_arena_bytes = prefill_graph_arena_bytes; + config.decode_graph_arena_bytes = decode_graph_arena_bytes; + config.evict_cuda_graph_cache_on_release = true; + const auto bound = bind_decoder_weights(weights->weights(), spec); + modules::QwenCausalDecodeRuntimeWeights bound_weights; + bound_weights.token_embedding = weights->weights().token_embedding; + bound_weights.stack = bound.stack; + bound_weights.final_norm = bound.final_norm; + bound_weights.lm_head = bound.lm_head; + reusable_decoder = std::make_unique( + *execution, config, bound_weights); + } + if (!embedding_graph) { + embedding_graph = std::make_unique(weights); + } + auto embeddings = embedding_graph->run(prompt.input_ids); + const int64_t width = spec.decoder.stack.hidden_size; + for (size_t i = 0; i < prompt.injection.positions.size(); ++i) { + std::copy_n(prompt.injection.values.data() + i * width, width, + embeddings.data() + prompt.injection.positions[i] * width); + } + const int64_t steps = static_cast(prompt.input_ids.size()); + const int64_t required = steps + max_new_tokens; + // Grow in bounded capacity buckets, keeping only one decode and one + // block-prefill graph. A new prompt clears KV on device, not via a + // host export/import of every layer. + const int64_t capacity = std::min(spec.max_position_embeddings, (required + 127) / 128 * 128); + const auto start = Clock::now(); + auto logits = reusable_decoder->prefill_embeddings_into_cache(embeddings, steps, capacity, 64).logits; + debug::timing_log_scalar("greedy_qwen_decoder.reusable.prefill_ms", engine::debug::elapsed_ms(start)); + std::vector out; + for (int64_t step = 0; step < max_new_tokens; ++step) { + const int32_t token = argmax_index(logits); + if (is_eos(spec, token)) { break; } + out.push_back(token); + if (step + 1 < max_new_tokens) { + logits = reusable_decoder->decode_token(token).logits; + } + } + return out; + } + }; GreedyQwenDecoderRuntime::GreedyQwenDecoderRuntime( @@ -633,7 +740,7 @@ GreedyQwenDecoderRuntime::GreedyQwenDecoderRuntime( GreedyQwenDecoderRuntime::~GreedyQwenDecoderRuntime() = default; -std::vector GreedyQwenDecoderRuntime::generate(const Prompt & prompt, int64_t max_new_tokens) { +std::vector GreedyQwenDecoderRuntime::generate(const Prompt & prompt, int64_t max_new_tokens, bool reuse_graphs) { const auto & spec = impl_->weights->spec(); if (prompt.input_ids.empty()) { throw std::runtime_error("greedy Qwen decoder prompt is empty"); @@ -651,6 +758,14 @@ std::vector GreedyQwenDecoderRuntime::generate(const Prompt & prompt, i static_cast(injection.values.size()) != injection.tokens * spec.decoder.stack.hidden_size) { throw std::runtime_error("greedy Qwen decoder injection shape does not match the prompt"); } + for (const auto position : injection.positions) { + if (position < 0 || position >= prompt_steps) { + throw std::runtime_error("greedy Qwen decoder injection position is out of range"); + } + } + if (reuse_graphs) { + return impl_->generate_reusing_graphs(prompt, max_new_tokens); + } if (impl_->prefill_graph == nullptr || !impl_->prefill_graph->matches(prompt_steps, injection.tokens)) { impl_->prefill_graph.reset(); impl_->prefill_graph = std::make_unique( diff --git a/tests/confucius4_r2t2/test_graph_reuse.cpp b/tests/confucius4_r2t2/test_graph_reuse.cpp new file mode 100644 index 000000000..15477484b --- /dev/null +++ b/tests/confucius4_r2t2/test_graph_reuse.cpp @@ -0,0 +1,117 @@ +#include "engine/community_models/confucius4_r2t2/assets.h" +#include "engine/community_models/confucius4_r2t2/audio_encoder.h" +#include "engine/community_models/confucius4_r2t2/thinker.h" +#include "engine/community_models/confucius4_r2t2/tokenizer_text.h" +#include "engine/community_models/confucius4_r2t2/types.h" +#include "engine/framework/core/execution_context.h" + +#include +#include +#include +#include +#include +#include + +#ifndef ENGINE_REPO_ROOT +#define ENGINE_REPO_ROOT "." +#endif + +int main(int argc, char ** argv) { + namespace model = engine::community_models::confucius4_r2t2; + std::filesystem::path path = std::filesystem::path(ENGINE_REPO_ROOT) / "models/Confucius4-R2T2"; + engine::core::BackendConfig backend; + backend.type = engine::core::BackendType::Cpu; + backend.threads = 8; + for (int i = 1; i + 1 < argc; i += 2) { + const std::string key = argv[i], value = argv[i + 1]; + if (key == "--model") { path = value; } + else if (key == "--backend" && value == "metal") { backend.type = engine::core::BackendType::Metal; } + else if (key != "--backend" || value != "cpu") { return 1; } + } + if (!std::filesystem::exists(path)) { + std::cerr << "SKIP: graph reuse parity requires a Confucius4-R2T2 checkpoint\n"; + return 125; + } + try { + auto assets = model::load_confucius4_r2t2_assets(path); + engine::core::ExecutionContext execution(backend); + model::R2T2ASRAudioEncoderRuntime exact(assets, execution, 128ull << 20, engine::assets::TensorStorageType::Native); + model::R2T2ASRAudioEncoderRuntime reused(assets, execution, 128ull << 20, engine::assets::TensorStorageType::Native); + // Exercise partial convolution chunks, capacity boundaries, and shrinking + // after growth. Nonzero deterministic input exposes padding leakage. + for (const int64_t frames : {32, 96, 100, 101, 199, 200, 201, 399, 400, 401, 799, 101, 32}) { + model::R2T2ASRAudioFeatures features; + features.frames = frames; + features.mel_bins = assets->config.audio_encoder.num_mel_bins; + features.encoder_tokens = model::confucius4_r2t2_audio_encoder_token_count(frames); + features.values.resize(frames * features.mel_bins); + for (size_t i = 0; i < features.values.size(); ++i) { + features.values[i] = 0.5f * std::sin(static_cast(i) * 0.037f); + } + const auto expected = exact.encode(features); + const auto actual = reused.encode(features, true); + if (expected.tokens != actual.tokens || expected.values.size() != actual.values.size()) { + throw std::runtime_error("encoder output shape changed"); + } + // Mirror the runtime's bucketing to know whether this run computed + // padded tokens at all. Without padded tokens the reusable graph + // must match the exact graph bit for bit. + const int64_t chunk_frames = assets->config.audio_encoder.n_window * 2; + const int64_t chunks = (frames + chunk_frames - 1) / chunk_frames; + const int64_t bucket_chunks = chunks <= 2 ? chunks : (chunks + 3) / 4 * 4; + const bool padded = frames >= chunk_frames && + model::confucius4_r2t2_audio_encoder_token_count(bucket_chunks * chunk_frames) != features.encoder_tokens; + float max_error = 0; + double error2 = 0, reference2 = 0; + for (size_t i = 0; i < actual.values.size(); ++i) { + const float error = std::abs(actual.values[i] - expected.values[i]); + if (!std::isfinite(error)) { throw std::runtime_error("nonfinite encoder output"); } + max_error = std::max(max_error, error); + error2 += error * error; + reference2 += expected.values[i] * expected.values[i]; + } + const double relative_rmse = std::sqrt(error2 / std::max(reference2, 1e-20)); + std::cout << "frames=" << frames << (padded ? " padded=yes" : " padded=no") + << " max_error=" << max_error << " relative_rmse=" << relative_rmse << '\n'; + if (!padded) { + if (max_error != 0.0F) { throw std::runtime_error("unpadded reusable graph differs from exact graph"); } + } else { + // Padded tokens change the reduction order inside softmax and + // attention value sums, so bit equality is impossible. Deep + // stacks amplify the kernel-order noise; the observed ceiling + // is about 2e-2 relative RMSE at 63 padded tokens. The real + // correctness bar is greedy decoder parity below plus the + // end-to-end streaming transcript check. + if (relative_rmse > 2e-2) { throw std::runtime_error("padded encoder drift exceeded the noise budget"); } + const auto again = reused.encode(features, true); + if (again.values != actual.values) { throw std::runtime_error("padded encoder output is not deterministic"); } + } + } + model::R2T2ASRTextTokenizer tokenizer(assets); + model::R2T2ASRThinkerRuntime thinker(assets, execution, 256ull << 20, 256ull << 20, 64ull << 20, + engine::assets::TensorStorageType::Native); + // Repeated prompts grow then shrink across prefill blocks and KV buckets. + for (const int64_t tokens : {4, 65, 129, 7, 65}) { + const auto prompt = tokenizer.build_prompt("", "English", tokens); + model::R2T2ASRAudioEmbeddings embeddings; + embeddings.tokens = tokens; + embeddings.hidden_size = assets->config.text_decoder.hidden_size; + embeddings.values.resize(tokens * embeddings.hidden_size); + for (size_t i = 0; i < embeddings.values.size(); ++i) { + embeddings.values[i] = 0.1f * std::cos(static_cast(i) * 0.013f); + } + model::R2T2ASRGenerationOptions options; + options.max_new_tokens = 8; + const auto expected = thinker.generate(prompt, embeddings, options); + options.reuse_graphs = true; + const auto actual = thinker.generate(prompt, embeddings, options); + if (expected.token_ids != actual.token_ids) { throw std::runtime_error("reused decoder token sequence changed"); } + std::cout << "injection_tokens=" << tokens << " decoder parity passed\n"; + } + std::cout << "PASS: graph reuse matches exact encoder and decoder after growth and shrink\n"; + return 0; + } catch (const std::exception & error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; + } +} From 116025757ff29d898eb3602f9c998e67db3dc663 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 12:58:50 +0900 Subject: [PATCH 7/9] fix(r2t2): guard Metal attention precision when reusing encoder graphs --- .../confucius4_r2t2/audio_encoder.h | 4 ++ .../confucius4_r2t2/audio_encoder.cpp | 13 +++- tests/confucius4_r2t2/test_graph_reuse.cpp | 72 +++++++++++++------ 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/include/engine/community_models/confucius4_r2t2/audio_encoder.h b/include/engine/community_models/confucius4_r2t2/audio_encoder.h index 5b1b2cda7..4e3033a14 100644 --- a/include/engine/community_models/confucius4_r2t2/audio_encoder.h +++ b/include/engine/community_models/confucius4_r2t2/audio_encoder.h @@ -22,6 +22,9 @@ class R2T2ASRAudioEncoderRuntime { assets::TensorStorageType weight_storage_type); ~R2T2ASRAudioEncoderRuntime(); + // Actual retained graph capacity, including capacity retained after shrink. + int64_t graph_capacity_frames() const noexcept { return graph_capacity_frames_; } + R2T2ASRAudioEmbeddings encode(const R2T2ASRAudioFeatures & features, bool reuse_graph = false); private: @@ -29,6 +32,7 @@ class R2T2ASRAudioEncoderRuntime { std::shared_ptr weights_; core::ExecutionContext * execution_ = nullptr; size_t graph_arena_bytes_ = 0; + int64_t graph_capacity_frames_ = 0; std::unique_ptr graph_; }; diff --git a/src/community_models/confucius4_r2t2/audio_encoder.cpp b/src/community_models/confucius4_r2t2/audio_encoder.cpp index 6670cef89..f850e0367 100644 --- a/src/community_models/confucius4_r2t2/audio_encoder.cpp +++ b/src/community_models/confucius4_r2t2/audio_encoder.cpp @@ -501,7 +501,12 @@ class R2T2ASRAudioEncoderGraph { const bool shape_matches = reusable_ && reusable ? frames >= chunk_frame_limit_ && frames <= frames_ : !reusable_ && !reusable && frames_ == frames; - return weights_.get() == &weights && shape_matches && backend_ == backend && compute_threads_ == std::max(1, threads); + // Metal switches its attention value matmul to half-input SIMD-group + // matrix multiplication at inner dimension 64. Do not move a shorter + // input across that precision boundary merely to reuse a larger graph. + const bool same_attention_kernel = backend_type_ != core::BackendType::Metal || + (confucius4_r2t2_audio_encoder_token_count(frames) < 64) == (output_tokens_ < 64); + return weights_.get() == &weights && shape_matches && same_attention_kernel && backend_ == backend && compute_threads_ == std::max(1, threads); } R2T2ASRAudioEmbeddings run(const R2T2ASRAudioFeatures & features) { @@ -627,7 +632,12 @@ R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeat // At most one graph is retained. Grow in four-chunk buckets after // the first two chunks, bounding padding overhead and memory. capacity = (chunks <= 2 ? chunks : (chunks + 3) / 4 * 4) * chunk_frames; + if (execution_->backend_type() == core::BackendType::Metal && features.encoder_tokens < 64 && + confucius4_r2t2_audio_encoder_token_count(capacity) >= 64) { + capacity = features.frames; + } } + graph_capacity_frames_ = 0; graph_.reset(); graph_ = std::make_unique( assets_, @@ -636,6 +646,7 @@ R2T2ASRAudioEmbeddings R2T2ASRAudioEncoderRuntime::encode(const R2T2ASRAudioFeat graph_arena_bytes_, capacity, reusable); + graph_capacity_frames_ = capacity; } else { debug::timing_log_scalar("confucius4_r2t2.audio_encoder.graph.build_ms", 0.0); diff --git a/tests/confucius4_r2t2/test_graph_reuse.cpp b/tests/confucius4_r2t2/test_graph_reuse.cpp index 15477484b..bd9bbe900 100644 --- a/tests/confucius4_r2t2/test_graph_reuse.cpp +++ b/tests/confucius4_r2t2/test_graph_reuse.cpp @@ -4,6 +4,8 @@ #include "engine/community_models/confucius4_r2t2/tokenizer_text.h" #include "engine/community_models/confucius4_r2t2/types.h" #include "engine/framework/core/execution_context.h" +#include "engine/framework/audio/wav_reader.h" +#include "engine/community_models/confucius4_r2t2/frontend_whisper.h" #include #include @@ -37,9 +39,29 @@ int main(int argc, char ** argv) { engine::core::ExecutionContext execution(backend); model::R2T2ASRAudioEncoderRuntime exact(assets, execution, 128ull << 20, engine::assets::TensorStorageType::Native); model::R2T2ASRAudioEncoderRuntime reused(assets, execution, 128ull << 20, engine::assets::TensorStorageType::Native); + model::R2T2ASRTextTokenizer tokenizer(assets); + model::R2T2ASRThinkerRuntime thinker(assets, execution, 256ull << 20, 256ull << 20, 64ull << 20, + engine::assets::TensorStorageType::Native); + auto check_joint = [&](const model::R2T2ASRAudioEmbeddings & expected, + const model::R2T2ASRAudioEmbeddings & actual, const std::string & language) { + const auto prompt = tokenizer.build_prompt("", language, expected.tokens); + model::R2T2ASRGenerationOptions options; + options.max_new_tokens = 32; + const auto reference = thinker.generate(prompt, expected, options); + options.reuse_graphs = true; + const auto candidate = thinker.generate(prompt, actual, options); + if (reference.token_ids.empty()) { + throw std::runtime_error("real-audio joint regression produced no tokens"); + } + if (reference.token_ids != candidate.token_ids) { + throw std::runtime_error("joint encoder/decoder token mismatch: exact=" + tokenizer.decode(reference.token_ids) + + " reused=" + tokenizer.decode(candidate.token_ids)); + } + std::cout << "joint parity tokens=" << reference.token_ids.size() << " language=" << language << '\n'; + }; // Exercise partial convolution chunks, capacity boundaries, and shrinking // after growth. Nonzero deterministic input exposes padding leakage. - for (const int64_t frames : {32, 96, 100, 101, 199, 200, 201, 399, 400, 401, 799, 101, 32}) { + for (const int64_t frames : {32, 96, 100, 101, 199, 200, 201, 399, 400, 401, 479, 480, 481, 487, 488, 489, 799, 199, 101, 32}) { model::R2T2ASRAudioFeatures features; features.frames = frames; features.mel_bins = assets->config.audio_encoder.num_mel_bins; @@ -53,14 +75,12 @@ int main(int argc, char ** argv) { if (expected.tokens != actual.tokens || expected.values.size() != actual.values.size()) { throw std::runtime_error("encoder output shape changed"); } - // Mirror the runtime's bucketing to know whether this run computed - // padded tokens at all. Without padded tokens the reusable graph - // must match the exact graph bit for bit. - const int64_t chunk_frames = assets->config.audio_encoder.n_window * 2; - const int64_t chunks = (frames + chunk_frames - 1) / chunk_frames; - const int64_t bucket_chunks = chunks <= 2 ? chunks : (chunks + 3) / 4 * 4; - const bool padded = frames >= chunk_frames && - model::confucius4_r2t2_audio_encoder_token_count(bucket_chunks * chunk_frames) != features.encoder_tokens; + const int64_t capacity = reused.graph_capacity_frames(); + const bool padded = model::confucius4_r2t2_audio_encoder_token_count(capacity) != features.encoder_tokens; + if (backend.type == engine::core::BackendType::Metal && features.encoder_tokens < 64 && + model::confucius4_r2t2_audio_encoder_token_count(capacity) >= 64) { + throw std::runtime_error("padding crossed the Metal attention precision boundary"); + } float max_error = 0; double error2 = 0, reference2 = 0; for (size_t i = 0; i < actual.values.size(); ++i) { @@ -72,24 +92,34 @@ int main(int argc, char ** argv) { } const double relative_rmse = std::sqrt(error2 / std::max(reference2, 1e-20)); std::cout << "frames=" << frames << (padded ? " padded=yes" : " padded=no") - << " max_error=" << max_error << " relative_rmse=" << relative_rmse << '\n'; + << " capacity_frames=" << capacity << " max_error=" << max_error << " relative_rmse=" << relative_rmse << '\n'; if (!padded) { if (max_error != 0.0F) { throw std::runtime_error("unpadded reusable graph differs from exact graph"); } } else { - // Padded tokens change the reduction order inside softmax and - // attention value sums, so bit equality is impossible. Deep - // stacks amplify the kernel-order noise; the observed ceiling - // is about 2e-2 relative RMSE at 63 padded tokens. The real - // correctness bar is greedy decoder parity below plus the - // end-to-end streaming transcript check. - if (relative_rmse > 2e-2) { throw std::runtime_error("padded encoder drift exceeded the noise budget"); } + // This bound is only an embedding drift alarm, not evidence of + // transcript equivalence or a diagnosis of the numerical cause. + // Joint encoder/decoder comparisons below check observable tokens. + if (relative_rmse > 2e-3) { throw std::runtime_error("padded encoder drift exceeded the noise budget"); } const auto again = reused.encode(features, true); if (again.values != actual.values) { throw std::runtime_error("padded encoder output is not deterministic"); } } } - model::R2T2ASRTextTokenizer tokenizer(assets); - model::R2T2ASRThinkerRuntime thinker(assets, execution, 256ull << 20, 256ull << 20, 64ull << 20, - engine::assets::TensorStorageType::Native); + // Real audio prefixes cover both automatic and forced language prompts, + // then shrink to exercise capacity selection and clearing previous inputs. + const auto wav = engine::audio::read_wav_f32(std::filesystem::path(ENGINE_REPO_ROOT) / "assets/resources/sample_16k.wav"); + model::R2T2ASRWhisperFrontend frontend(assets); + for (const double seconds : {1.01, 4.01, 4.89, 8.01, 1.99}) { + engine::runtime::AudioBuffer audio; + audio.sample_rate = wav.sample_rate; + audio.channels = wav.channels; + const size_t count = std::min(wav.samples.size(), static_cast(seconds * wav.sample_rate) * wav.channels); + audio.samples.assign(wav.samples.begin(), wav.samples.begin() + count); + const auto features = frontend.extract(audio); + const auto expected = exact.encode(features); + const auto actual = reused.encode(features, true); + check_joint(expected, actual, ""); + check_joint(expected, actual, "English"); + } // Repeated prompts grow then shrink across prefill blocks and KV buckets. for (const int64_t tokens : {4, 65, 129, 7, 65}) { const auto prompt = tokenizer.build_prompt("", "English", tokens); @@ -108,7 +138,7 @@ int main(int argc, char ** argv) { if (expected.token_ids != actual.token_ids) { throw std::runtime_error("reused decoder token sequence changed"); } std::cout << "injection_tokens=" << tokens << " decoder parity passed\n"; } - std::cout << "PASS: graph reuse matches exact encoder and decoder after growth and shrink\n"; + std::cout << "PASS: bounded encoder drift and joint token parity after growth and shrink\n"; return 0; } catch (const std::exception & error) { std::cerr << "FAIL: " << error.what() << '\n'; From b2db897f5666539e1565faaedaf1cfaf7635b3ad Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 12:58:50 +0900 Subject: [PATCH 8/9] docs(r2t2): clarify graph reuse validation and numerical limits --- docs/community_models/r2t2.md | 42 +++++++++++++++++++++++---------- tests/confucius4_r2t2/README.md | 14 +++++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index c1d8ec723..99d4156f8 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -188,18 +188,36 @@ the offline path keeps exact per-frame graphs): * Prompt token embeddings reuse a fixed-width lookup graph instead of a per-request build. -Measured on an M3 (Metal) over the 40 s golden clip (44 chunks): graph builds -drop from 44 encoder + 44 prefill + 44 decode to 8 + 13 + 3, total graph-build -time falls from about 1.06 s to 0.10 s, streaming wall time from 25.25 s to -23.67 s, and peak memory footprint from 7.32 GB to 7.14 GB. Transcripts are -identical with and without reuse. - -`test_confucius4_r2t2_graph_reuse` guards the semantics: encoder output is -bit-exact whenever the capacity bucket adds no padded tokens; padded runs are -held to a 2e-2 relative-RMSE noise budget (ggml reduction order changes with -sequence length, so bit equality is impossible once padding exists) and must -be deterministic across runs; decoder output must match the exact path token -for token across bucket growth and shrink. +The Metal encoder avoids padding an input with fewer than 64 tokens to a +capacity of 64 tokens or more, and rebuilds on shrink across that boundary. +The bundled Metal backend switches attention value multiplication to a +half-input SIMD-group matrix kernel at that size. A larger bucket can therefore +change arithmetic precision, not just floating-point summation order. + +Diagnostics on the M3 found identical convolution output, first-layer norm, +and Q/K/V for the tested inputs; the first difference appeared in attention +output when this boundary was crossed. Padding is not inherently inexact: +some padded shapes matched bit for bit. These findings describe the tested +backend and shapes, not a proof that all padding differences have one cause. + +`test_confucius4_r2t2_graph_reuse` reads the actual retained graph capacity, +including after shrink. It checks the Metal precision boundary, unpadded +output equality, padded repeatability, and a 2e-3 relative-RMSE drift alarm on +the synthetic fixture. That alarm is not a transcript-quality guarantee. +Joint tests pass exact encoder output through the exact decoder and reused +encoder output through the reused decoder, requiring identical token IDs for +real English audio prefixes with automatic and forced-English prompts. They +also check nonempty output, bucket boundaries, and growth followed by shrink. +The separate decoder-only test continues to cover injected prompt lengths. + +For a repeatable local benchmark, use the same checkpoint, backend, thread +count, and audio on both revisions. Count positive `graph.build_ms` entries +in `--log-file` output and compare `confucius4_r2t2.session.stream.wall_ms`, +CLI deltas, final text, and process memory measurements. The bundled English +sample is 14.0719 seconds (44 decode calls including the final flush at the +default 320 ms chunk size). A single timing comparison is not a general +performance guarantee; peak memory footprint and maximum resident set size +are different measurements. The port still recomputes the accumulated audio and prompt on each chunk; incremental encoder/prefill caching is a separate task. diff --git a/tests/confucius4_r2t2/README.md b/tests/confucius4_r2t2/README.md index 25cb28ffb..068dfb02a 100644 --- a/tests/confucius4_r2t2/README.md +++ b/tests/confucius4_r2t2/README.md @@ -63,3 +63,17 @@ The same binary can dump the family tokenizer for diffing against Hugging Face: build/macos-metal-release/bin/test_confucius4_r2t2_transcription \ --encode "language EnglishSome text 22,500" ``` + +## Graph reuse regression + +```bash +build/macos-metal-release/bin/test_confucius4_r2t2_graph_reuse --backend metal +``` + +This model-backed test compares actual encoder capacities after growth and +shrink, checks the Metal attention precision boundary, and compares exact +encoder + exact decoder against reused encoder + reused decoder on real audio +prefixes (automatic language and forced English). Decoder-only synthetic +injection tests remain separate. The embedding RMSE bound is a drift alarm; +observable token equality is checked independently. The test defaults to CPU +and accepts `--model` for a local safetensors directory or GGUF file. From d56446441ae1ae29d4dfc16999f12bff7de454c4 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 13:52:39 +0900 Subject: [PATCH 9/9] fix(r2t2): make the graph-reuse drift alarm backend-specific Tightening the padded-encoder alarm to 2e-3 alongside the Metal-only 64-token precision guard broke the test on CPU, its default backend: CPU has no kernel boundary to guard, and its fp32 reduction-order noise reaches about 1.9e-2 relative RMSE at 63 padded tokens, so the suite aborted at frames=101 before reaching the joint token checks. Split the alarm per backend: 2e-3 on Metal, where the 64-token capacity guard holds drift at or below 7.6e-4, and 2.5e-2 on CPU, above the measured 1.9e-2 ceiling. Verified on this machine: both backends pass the full suite, including joint encoder/decoder token parity on real audio prefixes with automatic and forced-English prompts. Also correct the r2t2 doc note that described the alarm as a single 2e-3 bound. --- docs/community_models/r2t2.md | 6 ++++-- tests/confucius4_r2t2/test_graph_reuse.cpp | 11 ++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index 99d4156f8..bc1939f31 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -202,8 +202,10 @@ backend and shapes, not a proof that all padding differences have one cause. `test_confucius4_r2t2_graph_reuse` reads the actual retained graph capacity, including after shrink. It checks the Metal precision boundary, unpadded -output equality, padded repeatability, and a 2e-3 relative-RMSE drift alarm on -the synthetic fixture. That alarm is not a transcript-quality guarantee. +output equality, padded repeatability, and a backend-specific +relative-RMSE drift alarm on the synthetic fixture (2e-3 on Metal under +the 64-token guard, 2.5e-2 on CPU, where fp32 reduction-order noise is +larger). That alarm is not a transcript-quality guarantee. Joint tests pass exact encoder output through the exact decoder and reused encoder output through the reused decoder, requiring identical token IDs for real English audio prefixes with automatic and forced-English prompts. They diff --git a/tests/confucius4_r2t2/test_graph_reuse.cpp b/tests/confucius4_r2t2/test_graph_reuse.cpp index bd9bbe900..4af6b8490 100644 --- a/tests/confucius4_r2t2/test_graph_reuse.cpp +++ b/tests/confucius4_r2t2/test_graph_reuse.cpp @@ -34,6 +34,12 @@ int main(int argc, char ** argv) { std::cerr << "SKIP: graph reuse parity requires a Confucius4-R2T2 checkpoint\n"; return 125; } + // Fixture-specific alarms measured independently on CPU and Metal: CPU + // padding noise is fp32 reduction order and reached about 1.9e-2 relative + // RMSE at 63 padded tokens, while Metal stays under the 64-token kernel + // guard and measured at most 7.6e-4. Neither tolerance substitutes for + // the exact joint token checks below. + const double encoder_drift_limit = backend.type == engine::core::BackendType::Cpu ? 2.5e-2 : 2e-3; try { auto assets = model::load_confucius4_r2t2_assets(path); engine::core::ExecutionContext execution(backend); @@ -99,7 +105,10 @@ int main(int argc, char ** argv) { // This bound is only an embedding drift alarm, not evidence of // transcript equivalence or a diagnosis of the numerical cause. // Joint encoder/decoder comparisons below check observable tokens. - if (relative_rmse > 2e-3) { throw std::runtime_error("padded encoder drift exceeded the noise budget"); } + if (relative_rmse > encoder_drift_limit) { + throw std::runtime_error("padded encoder drift " + std::to_string(relative_rmse) + + " exceeded backend fixture limit " + std::to_string(encoder_drift_limit)); + } const auto again = reused.encode(features, true); if (again.values != actual.values) { throw std::runtime_error("padded encoder output is not deterministic"); } }