From 2bb40bb36a00543c9be9b5f69332ba409bb4fe76 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Fri, 25 Sep 2026 15:11:41 -0400 Subject: [PATCH] Refactor ai config key/variation into dataclass --- .../src/launchdarkly_ai_server/__init__.py | 2 + .../evaluations/__init__.py | 10 +++- .../evaluations/module.py | 49 +++++++++---------- .../evaluations/types.py | 15 ++++++ packages/client/tests/test_evaluations_run.py | 38 +++++++------- 5 files changed, 68 insertions(+), 46 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 869d0ef0..24a7fe35 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -26,6 +26,7 @@ set_conversation_id_if_absent, ) from .evaluations import ( + AIConfig, Criterion, DatasetRow, EvalRunResult, @@ -184,6 +185,7 @@ "to_semconv_finish_reason", "VariationMeta", # evaluations + "AIConfig", "EvalRunResult", "Criterion", "DatasetRow", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py index bd3eb0a4..9cd942d9 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -11,10 +11,18 @@ ) from .criteria import Criterion, Judge, Scorer, SuccessDirection from .module import EvaluationsModule, init_evaluations -from .types import DatasetRow, EvalRunResult, GenerationConfig, RunSummary, Usage +from .types import ( + AIConfig, + DatasetRow, + EvalRunResult, + GenerationConfig, + RunSummary, + Usage, +) __all__ = [ "DEFAULT_BASE_URI", + "AIConfig", "Criterion", "DatasetRow", "EvalRunResult", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 1d34ef3f..4cc684aa 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -25,7 +25,7 @@ _provides_for, _segment, ) -from .types import EvalRunResult, GenerationConfig, RunSummary +from .types import AIConfig, EvalRunResult, GenerationConfig, RunSummary logger = logging.getLogger(__name__) @@ -128,8 +128,7 @@ async def run( dataset: str, handler: EvalHandler, generation: GenerationConfig | None = None, - ai_config: str | None = None, - variation: str | None = None, + ai_config: AIConfig | None = None, tools: Mapping[str, ToolImplementation] | None = None, criteria: list[Criterion] | None = None, judge_handlers: list[EvalHandler] | None = None, @@ -158,7 +157,7 @@ async def run( and a wider ``poll_interval_seconds``; both default to ``SUMMARY_POLL_TIMEOUT_SECONDS`` / ``SUMMARY_POLL_INTERVAL_SECONDS``. - Pass ``ai_config`` and ``variation`` to start from an existing AI Config + Pass ``ai_config`` to start from an existing AI Config variation instead of a hand-built ``generation``. Its model, provider, parameters, prompt and output format become the defaults, and anything set in ``generation`` overrides them field by field (``parameters`` @@ -180,18 +179,21 @@ async def run( poll_interval_seconds=poll_interval_seconds, poll_timeout_seconds=poll_timeout_seconds, ) - self._validate_config_source( - generation=generation, ai_config=ai_config, variation=variation - ) + self._validate_config_source(generation=generation, ai_config=ai_config) pinned_tool_versions: dict[str, int] = {} - if ai_config is not None and variation is not None: + config_label = "" + if ai_config is not None: + config_label = f"{ai_config.key!r}/{ai_config.variation!r}" ai_config_variation = await asyncio.to_thread( - self._runner._fetch_config_variation, project_key, ai_config, variation + self._runner._fetch_config_variation, + project_key, + ai_config.key, + ai_config.variation, ) generation = _merge_generation(ai_config_variation.generation, generation) if tools is None and ai_config_variation.tool_versions: raise EvaluationsError( - f"AI Config variation {ai_config!r}/{variation!r} uses tools " + f"AI Config variation {config_label} uses tools " "with no implementation: " + ", ".join( repr(name) for name in ai_config_variation.tool_versions @@ -226,10 +228,9 @@ async def run( resolved_tool = resolved_tools.get(tool_key) if resolved_tool is not None and resolved_tool.version != pinned_version: logger.warning( - "AI Config variation %r/%r pins tool %r at version %d; " + "AI Config variation %s pins tool %r at version %d; " "evaluating against the latest version %d.", - ai_config, - variation, + config_label, tool_key, pinned_version, resolved_tool.version, @@ -461,22 +462,20 @@ def _validate_run_args( def _validate_config_source( *, generation: GenerationConfig | None, - ai_config: str | None, - variation: str | None, + ai_config: AIConfig | None, ) -> None: """Require a generation source before any request is made.""" - if ai_config is None and variation is None: + if ai_config is None: if generation is None: raise EvaluationsError( - "Pass generation, or ai_config and variation to evaluate an " - "existing AI Config variation" + "Pass generation, or ai_config to evaluate an existing AI " + "Config variation" ) return - if ai_config is None: - raise EvaluationsError("variation requires ai_config") - if variation is None: - raise EvaluationsError("ai_config requires variation") - for name, value in (("ai_config", ai_config), ("variation", variation)): + for name, value in ( + ("ai_config.key", ai_config.key), + ("ai_config.variation", ai_config.variation), + ): if not value.strip(): raise EvaluationsError(f"{name} must not be blank") @@ -485,8 +484,8 @@ def _validate_generation(generation: GenerationConfig | None) -> GenerationConfi """Check the final generation settings, after any fetched variation is merged.""" if generation is None: raise EvaluationsError( - "Pass generation, or ai_config and variation to evaluate an " - "existing AI Config variation" + "Pass generation, or ai_config to evaluate an existing AI " + "Config variation" ) provider = generation.get("provider") model = generation.get("model") diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index 0eaf78c3..75af1aaa 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -67,6 +67,21 @@ class ResolvedTool: schema: dict[str, Any] = field(default_factory=dict) +@dataclass(frozen=True) +class AIConfig: + """Identifies an existing AI Config variation to seed an evaluation run. + + ``key`` is the AI Config key and ``variation`` is the variation key. The two + are only meaningful together, since a variation key is scoped to its config. + ``run()`` reads the variation's latest version from the management API. + This is a reference, not the config itself; see ``AiConfigRep`` for the + evaluated payload. + """ + + key: str + variation: str + + @dataclass class AIConfigVariation: """An AI Config variation read from the management API as run() defaults. diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 0752febd..eaa46d58 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -10,6 +10,7 @@ from launchdarkly_ai_server import create_handler from launchdarkly_ai_server.evaluations import ( + AIConfig, DatasetRow, EvaluationsError, HttpResponse, @@ -2340,8 +2341,7 @@ async def handler(config: dict[str, Any], *args: object) -> dict[str, Any]: key="eval-key", dataset="golden", handler=handler, - ai_config="support-agent", - variation="control", + ai_config=AIConfig(key="support-agent", variation="control"), ) assert result.passed is True @@ -2378,8 +2378,7 @@ async def handler(*args: object) -> dict[str, Any]: key="eval-key", dataset="golden", handler=handler, - ai_config="support-agent", - variation="control", + ai_config=AIConfig(key="support-agent", variation="control"), generation={ "model": "gpt-4o-mini", "parameters": {"temperature": 0.1}, @@ -2417,8 +2416,7 @@ async def test_variation_tools_without_implementations_fail_before_mutating_requ key="eval-key", dataset="golden", handler=successful_handler, - ai_config="support-agent", - variation="control", + ai_config=AIConfig(key="support-agent", variation="control"), ) assert [request["method"] for request in transport.requests] == ["GET", "GET"] @@ -2465,8 +2463,7 @@ async def handler(*args: object) -> dict[str, Any]: key="eval-key", dataset="golden", handler=handler, - ai_config="support-agent", - variation="control", + ai_config=AIConfig(key="support-agent", variation="control"), ) assert [request["method"] for request in transport.requests] == ["GET", "GET"] @@ -2485,8 +2482,7 @@ async def test_unknown_variation_fails_before_any_records_are_created() -> None: key="eval-key", dataset="golden", handler=successful_handler, - ai_config="support-agent", - variation="missing", + ai_config=AIConfig(key="support-agent", variation="missing"), ) assert [request["method"] for request in transport.requests] == ["GET"] @@ -2497,13 +2493,18 @@ async def test_unknown_variation_fails_before_any_records_are_created() -> None: ("source", "message"), [ ({}, "Pass generation"), - ({"variation": "control"}, "variation requires ai_config"), - ({"ai_config": "support-agent"}, "ai_config requires variation"), - ({"ai_config": " ", "variation": "control"}, "ai_config must not be blank"), + ( + {"ai_config": AIConfig(key=" ", variation="control")}, + "ai_config.key must not be blank", + ), + ( + {"ai_config": AIConfig(key="support-agent", variation="")}, + "ai_config.variation must not be blank", + ), ], ) async def test_config_source_is_validated_before_network_io( - source: dict[str, str], message: str + source: dict[str, AIConfig], message: str ) -> None: transport = SequencedTransport([]) evals = init_evaluations(api_token="token", transport=transport) @@ -2546,8 +2547,7 @@ async def handler(*args: object) -> dict[str, Any]: key="eval-key", dataset="golden", handler=handler, - ai_config="support-agent", - variation="control", + ai_config=AIConfig(key="support-agent", variation="control"), tools={"lookup_order": lookup_order}, ) @@ -2574,8 +2574,7 @@ async def test_non_string_model_config_key_fails_loudly( key="eval-key", dataset="golden", handler=successful_handler, - ai_config="support-agent", - variation="control", + ai_config=AIConfig(key="support-agent", variation="control"), ) assert [request["method"] for request in transport.requests] == ["GET"] @@ -2598,8 +2597,7 @@ async def test_variation_without_a_model_config_needs_an_explicit_provider( key="eval-key", dataset="golden", handler=successful_handler, - ai_config="support-agent", - variation="control", + ai_config=AIConfig(key="support-agent", variation="control"), ) assert [request["method"] for request in transport.requests] == ["GET"]