From a878e2933d6253ad035d69c187b7eb823222efa9 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Sun, 20 Sep 2026 04:26:18 -0600 Subject: [PATCH] Rebuild an ASR encoder graph when the cached one is too big ensure_graph() reused any cached graph at least as large as the request, and encode() zero-pads up to that capacity. The masks remove the padding from the result but not from the arithmetic, so the graph runs at its built size on every call -- quadratic in frames for self-attention. The capacity only ratchets upward, because only a larger request rebuilds. One long transcription therefore leaves the longest clip the process has ever seen as a floor under every later request, which is why the reported penalty scales with the previous request rather than being constant, and why unload_models clears it. parakeet_tdt already carried this fix and its measurements. Move the rule into engine::modules::asr_graph_capacity_usable() so nemotron_asr and hviske_asr get it too and the threshold lives in one place. Measured on nemotron_asr q8_0, CPU backend, 8 threads, 33s clip after a 262s clip: 25.7s -> 3.2s, matching the 3.1s fresh-server figure, with RSS after the sequence falling from 4377032 KiB to 1664236 KiB. The long request itself is unchanged. Output is unaffected either way: the same clip transcribes byte-identically on a matched graph and on a 262s oversized one, so this is a cost defect rather than a correctness one. Also bound relative_positional_encoding_cache_, which is keyed by frame count and holds (2 * frames - 1) * hidden floats per entry. Rebuilds are now common, so an unbounded map would grow without bound in a long-lived server. hviske_asr is fixed by inspection rather than measurement; the tolerance is inherited from parakeet_tdt's measurements rather than re-derived. Fixes #617 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QCCx4pMMex72qrNomPh5yL --- CMakeLists.txt | 3 + .../engine/framework/modules/asr_helpers.h | 17 ++++++ src/community_models/parakeet_tdt/encoder.cpp | 33 +++++------ src/framework/modules/asr_helpers.cpp | 7 +++ src/models/hviske_asr/encoder.cpp | 2 +- src/models/nemotron_asr/encoder.cpp | 16 ++++- tests/unittests/test_asr_graph_capacity.cpp | 59 +++++++++++++++++++ 7 files changed, 117 insertions(+), 20 deletions(-) create mode 100644 tests/unittests/test_asr_graph_capacity.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 769e550cf..2067e56d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2729,6 +2729,9 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST add_engine_unittest(safetensors_offsets_test tests/unittests/test_safetensors_offsets.cpp) add_test(NAME safetensors_offsets_test COMMAND safetensors_offsets_test) + add_engine_unittest(asr_graph_capacity_test tests/unittests/test_asr_graph_capacity.cpp) + add_test(NAME asr_graph_capacity_test COMMAND asr_graph_capacity_test) + add_engine_unittest(audio_chunking_test tests/unittests/test_audio_chunking.cpp) add_test(NAME audio_chunking_test COMMAND audio_chunking_test) add_engine_unittest(partial_text_test tests/unittests/test_partial_text.cpp) diff --git a/include/engine/framework/modules/asr_helpers.h b/include/engine/framework/modules/asr_helpers.h index c46d41e13..b0d9b265c 100644 --- a/include/engine/framework/modules/asr_helpers.h +++ b/include/engine/framework/modules/asr_helpers.h @@ -5,6 +5,23 @@ namespace engine::modules { +// True when a cached encoder graph built for `capacity_frames` may be reused +// for a request of `request_frames`. +// +// An ASR encoder graph runs at its built capacity no matter how short the real +// audio is -- encode() zero-pads up to it and masks the padding out of the +// result, not out of the arithmetic. So an oversized cached graph is paid for +// in full on every call, and with self-attention that cost is quadratic in +// frames. Reusing one indefinitely turns the largest request the process has +// ever seen into a floor under every later request. +// +// Rebuilding is a one-off cost of a few hundred ms, dominated by the positional +// projections, so it wins outright once the mismatch is more than a few +// percent. The tolerance keeps a stream of clips whose lengths wobble slightly +// from rebuilding on every call, while capping the wasted compute at roughly +// the same fraction. +bool asr_graph_capacity_usable(int64_t capacity_frames, int64_t request_frames); + std::vector make_asr_keep_mask(int64_t frames, int64_t valid_frames); void fill_asr_keep_mask(std::vector & out, int64_t frames, int64_t valid_frames); diff --git a/src/community_models/parakeet_tdt/encoder.cpp b/src/community_models/parakeet_tdt/encoder.cpp index b261bc216..2f4507ddd 100644 --- a/src/community_models/parakeet_tdt/encoder.cpp +++ b/src/community_models/parakeet_tdt/encoder.cpp @@ -377,6 +377,15 @@ const std::vector & ParakeetEncoderRuntime::relative_positional_encoding( if (cached != relative_positional_encoding_cache_.end()) { return cached->second; } + // Bounded: each entry is (2 * frames - 1) * hidden floats -- tens of MB at + // conversational lengths -- and the only caller is ensure_graph(), which + // rebuilds whenever the request size moves. A clip length that recurs hits + // the graph cache and never gets here, so keeping every size this process + // has ever seen would grow without bound to no benefit. + constexpr size_t kMaxCachedPositionalEncodings = 4; + if (relative_positional_encoding_cache_.size() >= kMaxCachedPositionalEncodings) { + relative_positional_encoding_cache_.clear(); + } auto inserted = relative_positional_encoding_cache_.emplace( frames, make_relative_positional_encoding(assets_->config.encoder.hidden_size, frames, assets_->config.encoder.max_position_embeddings)); @@ -387,26 +396,14 @@ void ParakeetEncoderRuntime::ensure_graph(int64_t input_frames, int64_t feature_ if (input_frames <= 0 || feature_dim <= 0) { throw std::runtime_error("Parakeet TDT encoder graph requires positive input shape"); } - // A cached graph is only reused if it is not much bigger than the request. - // - // The graph runs at its built capacity no matter how short the real audio - // is — encode() zero-pads up to it — so an oversized cached graph is paid - // for in full on every call. Measured on this encoder: a 7.4s clip costs - // 1018 ms on a matched graph and 10928 ms on a 60s-capacity one, while - // rebuilding costs ~400 ms once (dominated by the 24 positional - // projections; the allocation itself is ~0.4 ms). Rebuilding therefore wins - // outright whenever the mismatch is more than a few percent, and it wins by - // more with every subsequent call at the new size. - // - // The tolerance keeps the common case — a stream of clips whose lengths - // wobble slightly — from rebuilding on every call, while capping the wasted - // compute at roughly the same fraction. - constexpr double kMaxGraphOversizeRatio = 1.10; + // A cached graph is only reused if it is not much bigger than the request; + // see asr_graph_capacity_usable() for why. Measured on this encoder: a 7.4s + // clip costs 1018 ms on a matched graph and 10928 ms on a 60s-capacity one, + // while rebuilding costs ~400 ms once (dominated by the 24 positional + // projections; the allocation itself is ~0.4 ms). const bool capacity_usable = graph_ != nullptr && - graph_->input_frames >= input_frames && - static_cast(graph_->input_frames) <= - kMaxGraphOversizeRatio * static_cast(input_frames); + engine::modules::asr_graph_capacity_usable(graph_->input_frames, input_frames); if (capacity_usable && graph_->backend == execution_context_->backend() && graph_->feature_dim == feature_dim) { diff --git a/src/framework/modules/asr_helpers.cpp b/src/framework/modules/asr_helpers.cpp index ecaf49fa6..17e53bbf7 100644 --- a/src/framework/modules/asr_helpers.cpp +++ b/src/framework/modules/asr_helpers.cpp @@ -99,4 +99,11 @@ void fill_asr_stream_attention_bias( } } +bool asr_graph_capacity_usable(int64_t capacity_frames, int64_t request_frames) { + constexpr double kMaxGraphOversizeRatio = 1.10; + return capacity_frames >= request_frames && + static_cast(capacity_frames) <= + kMaxGraphOversizeRatio * static_cast(request_frames); +} + } // namespace engine::modules diff --git a/src/models/hviske_asr/encoder.cpp b/src/models/hviske_asr/encoder.cpp index 6cfae7f4f..5fa94017d 100644 --- a/src/models/hviske_asr/encoder.cpp +++ b/src/models/hviske_asr/encoder.cpp @@ -135,7 +135,7 @@ void HviskeEncoderRuntime::ensure_graph(int64_t input_frames, int64_t input_feat } if (graph_ != nullptr && graph_->backend == execution_context_->backend() && - graph_->input_frames >= input_frames && + engine::modules::asr_graph_capacity_usable(graph_->input_frames, input_frames) && graph_->input_features == input_features) { debug::timing_log_scalar("hviske_asr.encoder.graph_rebuild_ms", 0.0); debug::trace_log_scalar("hviske_asr.encoder.graph_cache_hit", true); diff --git a/src/models/nemotron_asr/encoder.cpp b/src/models/nemotron_asr/encoder.cpp index da72ff52b..a481dce46 100644 --- a/src/models/nemotron_asr/encoder.cpp +++ b/src/models/nemotron_asr/encoder.cpp @@ -510,6 +510,20 @@ const std::vector & NemotronEncoderRuntime::relative_positional_encoding( if (cached != relative_positional_encoding_cache_.end()) { return cached->second; } + // Bounded: each entry is (2 * frames - 1) * hidden floats -- tens of MB at + // conversational lengths -- and the offline path now rebuilds its graph + // whenever the request size moves, so keeping every size it has ever seen + // would grow without bound. + // + // Eviction is a coarse clear rather than an LRU, which can drop the entry + // the streaming path reuses on every chunk. That is acceptable because a + // streaming session asks for one stable key_frames, so on its own it never + // reaches the bound; only interleaved offline work at four different sizes + // can evict it, and the cost is one regeneration of a chunk-sized encoding. + constexpr size_t kMaxCachedPositionalEncodings = 4; + if (relative_positional_encoding_cache_.size() >= kMaxCachedPositionalEncodings) { + relative_positional_encoding_cache_.clear(); + } auto inserted = relative_positional_encoding_cache_.emplace( frames, make_relative_positional_encoding(1, assets_->config.encoder.hidden_size, frames, assets_->config.encoder.max_position_embeddings)); @@ -523,7 +537,7 @@ void NemotronEncoderRuntime::ensure_graph(int64_t input_frames, int64_t feature_ if (graph_ != nullptr && !graph_->streaming && graph_->backend == execution_context_->backend() && - graph_->input_frames >= input_frames && + engine::modules::asr_graph_capacity_usable(graph_->input_frames, input_frames) && graph_->feature_dim == feature_dim) { debug::timing_log_scalar("nemotron_asr.encoder.graph_rebuild_ms", 0.0); debug::trace_log_scalar("nemotron_asr.encoder.graph_cache_hit", true); diff --git a/tests/unittests/test_asr_graph_capacity.cpp b/tests/unittests/test_asr_graph_capacity.cpp new file mode 100644 index 000000000..7393d9db8 --- /dev/null +++ b/tests/unittests/test_asr_graph_capacity.cpp @@ -0,0 +1,59 @@ +#include "engine/framework/modules/asr_helpers.h" + +#include "test_assert.h" + +#include +#include + +namespace { + +using engine::modules::asr_graph_capacity_usable; +using engine::test::require; + +void test_exact_match_is_reusable() { + require(asr_graph_capacity_usable(1000, 1000), "a graph built for the request size is reusable"); +} + +void test_undersized_graph_is_not_reusable() { + require(!asr_graph_capacity_usable(999, 1000), "a graph smaller than the request cannot hold it"); + require(!asr_graph_capacity_usable(1, 1000), "a much smaller graph cannot hold the request"); +} + +void test_slightly_oversized_graph_is_reusable() { + // Clip lengths that wobble by a few percent should not force a rebuild on + // every call; the wasted compute is bounded by the same few percent. + require(asr_graph_capacity_usable(1050, 1000), "a 5% oversized graph is worth reusing"); + require(asr_graph_capacity_usable(1100, 1000), "the tolerance is inclusive at its edge"); +} + +void test_oversized_graph_is_rejected() { + // This is the issue #617 case: one long request must not leave a capacity + // behind that every later short request pays for. The encoder zero-pads up + // to the built capacity, so reuse here would cost ~8x on every call. + require(!asr_graph_capacity_usable(1101, 1000), "past the tolerance, rebuilding wins"); + require(!asr_graph_capacity_usable(8000, 1000), "a graph 8x the request must be rebuilt"); +} + +void test_zero_sizes_are_handled_without_special_casing() { + // Every caller rejects a non-positive frame count before reaching here, so + // this pins down behaviour rather than guarding a reachable path. + require(!asr_graph_capacity_usable(0, 1000), "an empty graph cannot hold a request"); + require(asr_graph_capacity_usable(0, 0), "zero capacity trivially holds a zero request"); +} + +} // namespace + +int main() { + try { + test_exact_match_is_reusable(); + test_undersized_graph_is_not_reusable(); + test_slightly_oversized_graph_is_reusable(); + test_oversized_graph_is_rejected(); + test_zero_sizes_are_handled_without_special_casing(); + std::cout << "asr_graph_capacity_test passed\n"; + } catch (const std::exception & ex) { + std::cerr << "asr_graph_capacity_test failed: " << ex.what() << "\n"; + return 1; + } + return 0; +}