diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 454ae5f30..8103a3bb6 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -67,13 +67,10 @@ pub async fn run( let overhead = run_started.elapsed(); metrics::record_routing_overhead(&algorithm_name, overhead); - let selected_model_id = outcome.selected_model_id; + let selected_model_id = outcome.selected_model_id()?.clone(); let (result, answer_duration) = if let Some(response) = outcome.response { (Ok(response), None) } else { - let mut models = Vec::with_capacity(1 + outcome.fallback_models.len()); - models.push(selected_model_id.clone()); - models.extend(outcome.fallback_models); let answer_started = Instant::now(); let observe = |observation| { if let Some(observer) = &observer { @@ -84,7 +81,7 @@ pub async fn run( &clients, &algorithm_name, &outcome.request, - &models, + &outcome.selected_model_ids, CallPhase::Completion, &observe, ) @@ -119,8 +116,8 @@ pub async fn decide( serve(routing_clients.clone(), call, None) }) .await?; - outcome.request = - clients.prepare_completion_request(outcome.request, &outcome.selected_model_id); + let selected_model_id = outcome.selected_model_id()?.clone(); + outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id); Ok(outcome) } @@ -750,7 +747,10 @@ mod tests { ) .await?; - assert_eq!(outcome.fallback_models, [ModelId::from("strong")]); + assert_eq!( + outcome.selected_model_ids, + [ModelId::from("weak"), ModelId::from("strong")] + ); assert_eq!(instruction_text(&outcome.request), ["weak prompt"]); Ok(()) } diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index 7f6dd9eaf..2146d96fb 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -641,8 +641,10 @@ mod tests { tokio::pin!(stream); while let Some(step) = stream.next().await { if let crate::Step::Done(outcome) = step? { - assert_eq!(outcome.selected_model_id, ModelId::from("mid")); - assert_eq!(outcome.fallback_models, target_set(&["weak", "strong"])); + assert_eq!( + outcome.selected_model_ids, + target_set(&["mid", "weak", "strong"]) + ); assert_eq!(outcome.request.llm_request.model.as_deref(), Some("mid")); assert!(outcome.response.is_none()); return Ok(()); diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 47ddfb668..9c89a4553 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -62,10 +62,8 @@ impl CallModel { /// The terminal result of routing. pub struct RoutingOutcome { - /// The model selected by the algorithm and tried first by the client. - pub selected_model_id: ModelId, - /// Additional models the client may try in order after an eligible failure. - pub fallback_models: Vec, + /// Models selected by the algorithm, ordered best model first. + pub selected_model_ids: Vec, /// The request after all routing-time rewrites, stamped with the selected model. pub request: Request, /// A response produced while routing, or `None` when the client must make the answer call. @@ -73,6 +71,12 @@ pub struct RoutingOutcome { } impl RoutingOutcome { + /// The model the algorithm recommends, the best model for this request. + /// `LibsyError::NoTargets` if the algorithm selected no models, which should be impossible. + pub fn selected_model_id(&self) -> Result<&ModelId> { + self.selected_model_ids.first().ok_or(LibsyError::NoTargets) + } + /// The decision is that client should send this `request`. The `selected_model_id` /// will be written into it by this function. /// If that fails client should try the `fallback_models` in order. @@ -82,9 +86,11 @@ impl RoutingOutcome { mut request: Request, ) -> Self { request.llm_request.model = Some(selected_model_id.to_string()); + let mut selected_model_ids = Vec::with_capacity(1 + fallback_models.len()); + selected_model_ids.push(selected_model_id); + selected_model_ids.extend(fallback_models); Self { - selected_model_id, - fallback_models, + selected_model_ids, request, response: None, } @@ -95,8 +101,7 @@ impl RoutingOutcome { pub fn answered(selected_model_id: ModelId, mut request: Request, response: Response) -> Self { request.llm_request.model = Some(selected_model_id.to_string()); Self { - selected_model_id, - fallback_models: Vec::new(), + selected_model_ids: vec![selected_model_id], request, response: Some(response), } @@ -194,7 +199,7 @@ impl Driver { let selected_model = result .as_ref() .ok() - .map(|outcome| outcome.selected_model_id.clone()); + .and_then(|outcome| outcome.selected_model_id().ok().cloned()); let step = result.map(|outcome| Step::Done(Box::new(outcome))); self.step_tx .send(step) @@ -472,16 +477,15 @@ mod tests { request(), ); - assert_eq!(outcome.selected_model_id, "selected"); assert_eq!( - outcome.fallback_models, - target_set(&["fallback-one", "fallback-two"]) + outcome.selected_model_ids, + target_set(&["selected", "fallback-one", "fallback-two"]) ); assert_eq!(outcome.request.model_id().as_deref(), Some("selected")); assert!(outcome.response.is_none()); let outcome = RoutingOutcome::route_to("only".into(), Vec::new(), request()); - assert!(outcome.fallback_models.is_empty()); + assert_eq!(outcome.selected_model_ids, target_set(&["only"])); let outcome = RoutingOutcome::answered( "answered".into(), @@ -492,9 +496,8 @@ mod tests { }, ); - assert_eq!(outcome.selected_model_id, "answered"); + assert_eq!(outcome.selected_model_ids, target_set(&["answered"])); assert_eq!(outcome.request.model_id().as_deref(), Some("answered")); - assert!(outcome.fallback_models.is_empty()); assert_eq!( outcome .response diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs index 3fdb8c38b..aeb570988 100644 --- a/crates/libsy/src/core/testing.rs +++ b/crates/libsy/src/core/testing.rs @@ -53,7 +53,7 @@ pub(crate) async fn test_drive( fulfill(Arc::clone(&routing_serve), call) }) .await?; - let selected_model = outcome.selected_model_id.clone(); + let selected_model = outcome.selected_model_id()?.clone(); let response = match outcome.response { Some(response) => response, None => serve diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index e2d6a4eaf..0571a3e12 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -517,24 +517,17 @@ impl PyModelCall { /// The terminal routing selection, rewritten request, and optional existing response. #[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)] struct PyRoutingOutcome { - selected_model_id: String, - fallback_models: Vec, + selected_model_ids: Vec, request: Py, response: Option>, } #[pymethods] impl PyRoutingOutcome { - /// The model selected by the algorithm and tried first by the host. + /// Models selected by the algorithm, ordered best model first. #[getter] - fn selected_model_id(&self) -> &str { - &self.selected_model_id - } - - /// Additional models the host may try in order after an eligible failure. - #[getter] - fn fallback_models(&self) -> Vec { - self.fallback_models.clone() + fn selected_model_ids(&self) -> Vec { + self.selected_model_ids.clone() } /// The normalized request after routing-time rewrites. @@ -679,8 +672,7 @@ fn step_to_python(step: RustStep) -> PyResult { }), RustStep::Done(outcome) => { let RoutingOutcome { - selected_model_id, - fallback_models, + selected_model_ids, request, response, } = *outcome; @@ -689,8 +681,7 @@ fn step_to_python(step: RustStep) -> PyResult { outcome: Py::new( py, PyRoutingOutcome { - selected_model_id: selected_model_id.to_string(), - fallback_models: fallback_models + selected_model_ids: selected_model_ids .iter() .map(ToString::to_string) .collect(), diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs index bcd175344..36ecb6410 100644 --- a/crates/switchyard-runner/src/runner.rs +++ b/crates/switchyard-runner/src/runner.rs @@ -99,13 +99,10 @@ impl Runner { ) -> Option { let route = self.route(model.as_str())?; let resolve = |selected: &ModelId| route.decision_target(selected); + let mut model_ids = outcome.selected_model_ids.iter(); Some(DecisionDescription { - selected: resolve(&outcome.selected_model_id)?, - fallbacks: outcome - .fallback_models - .iter() - .map(resolve) - .collect::>>()?, + selected: resolve(model_ids.next()?)?, + fallbacks: model_ids.map(resolve).collect::>>()?, }) } } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index eea1c7a2e..844931831 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -723,7 +723,7 @@ async fn decision( match encode_aggregated_response( &aggregate, input_format, - Some(outcome.selected_model_id.as_str()), + outcome.selected_model_id().ok().map(ModelId::as_str), ) { Ok(response) => Some(response), Err(error) => return server_error(error.to_string()), diff --git a/examples/libsy.py b/examples/libsy.py index 471ab5b55..fd6b8b684 100644 --- a/examples/libsy.py +++ b/examples/libsy.py @@ -68,10 +68,10 @@ async def main() -> None: case Step.CallModel(call): call.respond(await client.call(call.request, call.models[0])) case Step.Done(outcome): - print("Decision:", outcome.selected_model_id) + print("Decision:", outcome.selected_model_ids[0]) response = outcome.response or await client.call( outcome.request, - outcome.selected_model_id, + outcome.selected_model_ids[0], ) match response: case LlmResponse.Agg(aggregate_response): diff --git a/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py b/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py index 1532204a1..333d9a80e 100644 --- a/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py +++ b/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py @@ -181,7 +181,7 @@ async def run(self, context: RoutingContext) -> RoutingContext: "Switchyard algorithm produced a response while routing, " "which a LiteLLM routing plugin cannot return" ) - selected = outcome.selected_model_id + selected = outcome.selected_model_ids[0] if selected not in candidates: raise ValueError( f"Switchyard selected {selected!r}, which is not in LiteLLM's " @@ -197,7 +197,7 @@ async def run(self, context: RoutingContext) -> RoutingContext: ] context.signals["switchyard"] = { "selected_model_id": selected, - "fallback_models": outcome.fallback_models, + "fallback_models": outcome.selected_model_ids[1:], } if request_patch["set"] or request_patch.get("remove"): context.signals["switchyard"]["request_patch"] = request_patch diff --git a/examples/litellm/tests/unit/test_switchyard_routing_plugin.py b/examples/litellm/tests/unit/test_switchyard_routing_plugin.py index 3038f0bb5..e0cca081c 100644 --- a/examples/litellm/tests/unit/test_switchyard_routing_plugin.py +++ b/examples/litellm/tests/unit/test_switchyard_routing_plugin.py @@ -166,15 +166,14 @@ async def test_litellm_conversion_preserves_stage_tool_signal_input() -> None: # ToolSignals reads only normalized messages, so exact equality protects every signal input. assert _request(litellm_messages)["messages"] == original_messages assert direct_outcome is not None - assert direct_outcome.selected_model_id == SOL - assert direct_outcome.fallback_models == [TERRA] + assert direct_outcome.selected_model_ids == [SOL, TERRA] context = routing_context(litellm_messages) await stage_plugin().run(context) assert context.signals["switchyard"] == { - "selected_model_id": direct_outcome.selected_model_id, - "fallback_models": direct_outcome.fallback_models, + "selected_model_id": direct_outcome.selected_model_ids[0], + "fallback_models": direct_outcome.selected_model_ids[1:], } diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index cb15172cc..1589d60ff 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -114,10 +114,7 @@ def fail(self, error: BaseException) -> None: ... @final class RoutingOutcome: @property - def selected_model_id(self) -> str: ... - - @property - def fallback_models(self) -> list[str]: ... + def selected_model_ids(self) -> list[str]: ... @property def request(self) -> dict[str, object]: ... diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index 07dd3cda7..02ac614e4 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -80,10 +80,10 @@ async def run_algorithm( if outcome.response is not None: match outcome.response: case LlmResponse.Agg(response): - return outcome.selected_model_id, response + return outcome.selected_model_ids[0], response case LlmResponse.Stream(_): raise AssertionError("test helper expected an aggregate response") - candidates = [outcome.selected_model_id, *outcome.fallback_models] + candidates = outcome.selected_model_ids for index, target in enumerate(candidates): candidate_request = {**outcome.request, "model": target} client = (clients or {})[target] @@ -93,7 +93,7 @@ async def run_algorithm( if index + 1 == len(candidates): raise else: - return outcome.selected_model_id, response + return outcome.selected_model_ids[0], response raise AssertionError("algorithm stream ended without an outcome") @@ -111,8 +111,7 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response() assert variants == ["done"] assert outcome is not None - assert outcome.selected_model_id == "fast" - assert outcome.fallback_models == [] + assert outcome.selected_model_ids == ["fast"] assert outcome.response is None response = await client.call(outcome.request) assert client.calls[0]["model"] == "fast" @@ -156,7 +155,7 @@ async def events() -> AsyncIterator[dict[str, object]]: outcome = done assert outcome is not None - assert outcome.selected_model_id == "model-b" + assert outcome.selected_model_ids == ["model-b", "model-a"] async def test_classifier_config_accepts_a_prompt_override() -> None: