Rebuild an ASR encoder graph when the cached one is too big

CMakeLists.txt

@@ -2729,6 +2729,9 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
         add_engine_unittest(safetensors_offsets_test tests/unittests/test_safetensors_offsets.cpp)
         add_test(NAME safetensors_offsets_test COMMAND safetensors_offsets_test)
 
+        add_engine_unittest(asr_graph_capacity_test tests/unittests/test_asr_graph_capacity.cpp)
+        add_test(NAME asr_graph_capacity_test COMMAND asr_graph_capacity_test)
+
         add_engine_unittest(audio_chunking_test tests/unittests/test_audio_chunking.cpp)
         add_test(NAME audio_chunking_test COMMAND audio_chunking_test)
         add_engine_unittest(partial_text_test tests/unittests/test_partial_text.cpp)

include/engine/framework/modules/asr_helpers.h

@@ -5,6 +5,23 @@
 
 namespace engine::modules {
 
+// True when a cached encoder graph built for `capacity_frames` may be reused
+// for a request of `request_frames`.
+//
+// An ASR encoder graph runs at its built capacity no matter how short the real
+// audio is -- encode() zero-pads up to it and masks the padding out of the
+// result, not out of the arithmetic. So an oversized cached graph is paid for
+// in full on every call, and with self-attention that cost is quadratic in
+// frames. Reusing one indefinitely turns the largest request the process has
+// ever seen into a floor under every later request.
+//
+// Rebuilding is a one-off cost of a few hundred ms, dominated by the positional
+// projections, so it wins outright once the mismatch is more than a few
+// percent. The tolerance keeps a stream of clips whose lengths wobble slightly
+// from rebuilding on every call, while capping the wasted compute at roughly
+// the same fraction.
+bool asr_graph_capacity_usable(int64_t capacity_frames, int64_t request_frames);
+
 std::vector<int32_t> make_asr_keep_mask(int64_t frames, int64_t valid_frames);
 
 void fill_asr_keep_mask(std::vector<int32_t> & out, int64_t frames, int64_t valid_frames);

src/community_models/parakeet_tdt/encoder.cpp

@@ -377,6 +377,15 @@ const std::vector<float> & ParakeetEncoderRuntime::relative_positional_encoding(
     if (cached != relative_positional_encoding_cache_.end()) {
         return cached->second;
     }
+    // Bounded: each entry is (2 * frames - 1) * hidden floats -- tens of MB at
+    // conversational lengths -- and the only caller is ensure_graph(), which
+    // rebuilds whenever the request size moves. A clip length that recurs hits
+    // the graph cache and never gets here, so keeping every size this process
+    // has ever seen would grow without bound to no benefit.
+    constexpr size_t kMaxCachedPositionalEncodings = 4;
+    if (relative_positional_encoding_cache_.size() >= kMaxCachedPositionalEncodings) {
+        relative_positional_encoding_cache_.clear();
+    }
     auto inserted = relative_positional_encoding_cache_.emplace(
         frames,
         make_relative_positional_encoding(assets_->config.encoder.hidden_size, frames, assets_->config.encoder.max_position_embeddings));
@@ -387,26 +396,14 @@ void ParakeetEncoderRuntime::ensure_graph(int64_t input_frames, int64_t feature_
     if (input_frames <= 0 || feature_dim <= 0) {
         throw std::runtime_error("Parakeet TDT encoder graph requires positive input shape");
     }
-    // A cached graph is only reused if it is not much bigger than the request.
-    //
-    // The graph runs at its built capacity no matter how short the real audio
-    // is — encode() zero-pads up to it — so an oversized cached graph is paid
-    // for in full on every call. Measured on this encoder: a 7.4s clip costs
-    // 1018 ms on a matched graph and 10928 ms on a 60s-capacity one, while
-    // rebuilding costs ~400 ms once (dominated by the 24 positional
-    // projections; the allocation itself is ~0.4 ms). Rebuilding therefore wins
-    // outright whenever the mismatch is more than a few percent, and it wins by
-    // more with every subsequent call at the new size.
-    //
-    // The tolerance keeps the common case — a stream of clips whose lengths
-    // wobble slightly — from rebuilding on every call, while capping the wasted
-    // compute at roughly the same fraction.
-    constexpr double kMaxGraphOversizeRatio = 1.10;
+    // A cached graph is only reused if it is not much bigger than the request;
+    // see asr_graph_capacity_usable() for why. Measured on this encoder: a 7.4s
+    // clip costs 1018 ms on a matched graph and 10928 ms on a 60s-capacity one,
+    // while rebuilding costs ~400 ms once (dominated by the 24 positional
+    // projections; the allocation itself is ~0.4 ms).
     const bool capacity_usable =
         graph_ != nullptr &&
-        graph_->input_frames >= input_frames &&
-        static_cast<double>(graph_->input_frames) <=
-            kMaxGraphOversizeRatio * static_cast<double>(input_frames);
+        engine::modules::asr_graph_capacity_usable(graph_->input_frames, input_frames);
     if (capacity_usable &&
         graph_->backend == execution_context_->backend() &&
         graph_->feature_dim == feature_dim) {

src/framework/modules/asr_helpers.cpp

@@ -99,4 +99,11 @@ void fill_asr_stream_attention_bias(
     }
 }
 
+bool asr_graph_capacity_usable(int64_t capacity_frames, int64_t request_frames) {
+    constexpr double kMaxGraphOversizeRatio = 1.10;
+    return capacity_frames >= request_frames &&
+        static_cast<double>(capacity_frames) <=
+            kMaxGraphOversizeRatio * static_cast<double>(request_frames);
+}
+
 }  // namespace engine::modules

src/models/hviske_asr/encoder.cpp

@@ -135,7 +135,7 @@ void HviskeEncoderRuntime::ensure_graph(int64_t input_frames, int64_t input_feat
     }
     if (graph_ != nullptr &&
         graph_->backend == execution_context_->backend() &&
-        graph_->input_frames >= input_frames &&
+        engine::modules::asr_graph_capacity_usable(graph_->input_frames, input_frames) &&
         graph_->input_features == input_features) {
         debug::timing_log_scalar("hviske_asr.encoder.graph_rebuild_ms", 0.0);
         debug::trace_log_scalar("hviske_asr.encoder.graph_cache_hit", true);

src/models/nemotron_asr/encoder.cpp

@@ -510,6 +510,20 @@ const std::vector<float> & NemotronEncoderRuntime::relative_positional_encoding(
     if (cached != relative_positional_encoding_cache_.end()) {
         return cached->second;
     }
+    // Bounded: each entry is (2 * frames - 1) * hidden floats -- tens of MB at
+    // conversational lengths -- and the offline path now rebuilds its graph
+    // whenever the request size moves, so keeping every size it has ever seen
+    // would grow without bound.
+    //
+    // Eviction is a coarse clear rather than an LRU, which can drop the entry
+    // the streaming path reuses on every chunk. That is acceptable because a
+    // streaming session asks for one stable key_frames, so on its own it never
+    // reaches the bound; only interleaved offline work at four different sizes
+    // can evict it, and the cost is one regeneration of a chunk-sized encoding.
+    constexpr size_t kMaxCachedPositionalEncodings = 4;
+    if (relative_positional_encoding_cache_.size() >= kMaxCachedPositionalEncodings) {
+        relative_positional_encoding_cache_.clear();
+    }
     auto inserted = relative_positional_encoding_cache_.emplace(
         frames,
         make_relative_positional_encoding(1, assets_->config.encoder.hidden_size, frames, assets_->config.encoder.max_position_embeddings));
@@ -523,7 +537,7 @@ void NemotronEncoderRuntime::ensure_graph(int64_t input_frames, int64_t feature_
     if (graph_ != nullptr &&
         !graph_->streaming &&
         graph_->backend == execution_context_->backend() &&
-        graph_->input_frames >= input_frames &&
+        engine::modules::asr_graph_capacity_usable(graph_->input_frames, input_frames) &&
         graph_->feature_dim == feature_dim) {
         debug::timing_log_scalar("nemotron_asr.encoder.graph_rebuild_ms", 0.0);
         debug::trace_log_scalar("nemotron_asr.encoder.graph_cache_hit", true);

tests/unittests/test_asr_graph_capacity.cpp

@@ -0,0 +1,59 @@
+#include "engine/framework/modules/asr_helpers.h"
+
+#include "test_assert.h"
+
+#include <iostream>
+#include <stdexcept>
+
+namespace {
+
+using engine::modules::asr_graph_capacity_usable;
+using engine::test::require;
+
+void test_exact_match_is_reusable() {
+    require(asr_graph_capacity_usable(1000, 1000), "a graph built for the request size is reusable");
+}
+
+void test_undersized_graph_is_not_reusable() {
+    require(!asr_graph_capacity_usable(999, 1000), "a graph smaller than the request cannot hold it");
+    require(!asr_graph_capacity_usable(1, 1000), "a much smaller graph cannot hold the request");
+}
+
+void test_slightly_oversized_graph_is_reusable() {
+    // Clip lengths that wobble by a few percent should not force a rebuild on
+    // every call; the wasted compute is bounded by the same few percent.
+    require(asr_graph_capacity_usable(1050, 1000), "a 5% oversized graph is worth reusing");
+    require(asr_graph_capacity_usable(1100, 1000), "the tolerance is inclusive at its edge");
+}
+
+void test_oversized_graph_is_rejected() {
+    // This is the issue #617 case: one long request must not leave a capacity
+    // behind that every later short request pays for. The encoder zero-pads up
+    // to the built capacity, so reuse here would cost ~8x on every call.
+    require(!asr_graph_capacity_usable(1101, 1000), "past the tolerance, rebuilding wins");
+    require(!asr_graph_capacity_usable(8000, 1000), "a graph 8x the request must be rebuilt");
+}
+
+void test_zero_sizes_are_handled_without_special_casing() {
+    // Every caller rejects a non-positive frame count before reaching here, so
+    // this pins down behaviour rather than guarding a reachable path.
+    require(!asr_graph_capacity_usable(0, 1000), "an empty graph cannot hold a request");
+    require(asr_graph_capacity_usable(0, 0), "zero capacity trivially holds a zero request");
+}
+
+}  // namespace
+
+int main() {
+    try {
+        test_exact_match_is_reusable();
+        test_undersized_graph_is_not_reusable();
+        test_slightly_oversized_graph_is_reusable();
+        test_oversized_graph_is_rejected();
+        test_zero_sizes_are_handled_without_special_casing();
+        std::cout << "asr_graph_capacity_test passed\n";
+    } catch (const std::exception & ex) {
+        std::cerr << "asr_graph_capacity_test failed: " << ex.what() << "\n";
+        return 1;
+    }
+    return 0;
+}