From a2d5e4ccbd1f3af32b50e669b6f94542b71654e4 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Thu, 24 Sep 2026 11:57:39 -0400 Subject: [PATCH 1/4] Support pulling in configs for evals from code --- .../evaluations/module.py | 137 +++++++- .../evaluations/runner.py | 129 ++++++++ .../evaluations/types.py | 15 + packages/client/tests/test_evaluations_run.py | 307 ++++++++++++++++++ 4 files changed, 574 insertions(+), 14 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 80d22ece..1d34ef3f 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -7,7 +7,7 @@ import os import time from collections.abc import Mapping -from typing import Any +from typing import Any, cast from ..lifecycle import get_client, init_client from .api import ( @@ -63,6 +63,35 @@ def _is_terminal_summary(summary: RunSummary) -> bool: ) +def _merge_generation( + base: GenerationConfig, override: GenerationConfig | None +) -> GenerationConfig: + """Layer a caller's generation settings over a fetched variation's. + + Keys the caller sets replace the fetched ones, except ``parameters``, which + merge key by key so overriding ``temperature`` keeps a fetched + ``max_tokens``. ``instructions`` and ``messages`` are one prompt slot: + supplying either discards both fetched values, so a caller swapping an + agent prompt for a message list does not trip the mutual-exclusion check. + """ + merged: dict[str, Any] = dict(base) + if not override: + return cast(GenerationConfig, merged) + if "instructions" in override or "messages" in override: + merged.pop("instructions", None) + merged.pop("messages", None) + for field_name, value in override.items(): + if field_name == "parameters" and isinstance(value, Mapping): + base_parameters = merged.get("parameters") + merged["parameters"] = { + **(base_parameters if isinstance(base_parameters, Mapping) else {}), + **value, + } + else: + merged[field_name] = value + return cast(GenerationConfig, merged) + + class EvaluationsModule: """Entry point for running LaunchDarkly evaluations from customer code.""" @@ -98,7 +127,9 @@ async def run( key: str, dataset: str, handler: EvalHandler, - generation: GenerationConfig, + generation: GenerationConfig | None = None, + ai_config: str | None = None, + variation: str | None = None, tools: Mapping[str, ToolImplementation] | None = None, criteria: list[Criterion] | None = None, judge_handlers: list[EvalHandler] | None = None, @@ -126,6 +157,15 @@ async def run( this method. Large datasets may need a longer ``poll_timeout_seconds`` and a wider ``poll_interval_seconds``; both default to ``SUMMARY_POLL_TIMEOUT_SECONDS`` / ``SUMMARY_POLL_INTERVAL_SECONDS``. + + Pass ``ai_config`` and ``variation`` to start from an existing AI Config + variation instead of a hand-built ``generation``. Its model, provider, + parameters, prompt and output format become the defaults, and anything + set in ``generation`` overrides them field by field (``parameters`` + merge key by key). When ``tools`` is omitted the variation's tools are + used, so each needs an implementation; pass ``tools`` to replace the + set. When ``criteria`` is omitted the variation's attached judges run; + pass ``criteria`` (even ``[]``) to replace them. """ if poll_interval_seconds is None: poll_interval_seconds = SUMMARY_POLL_INTERVAL_SECONDS @@ -136,11 +176,34 @@ async def run( key=key, dataset=dataset, handler=handler, - generation=generation, concurrency=concurrency, poll_interval_seconds=poll_interval_seconds, poll_timeout_seconds=poll_timeout_seconds, ) + self._validate_config_source( + generation=generation, ai_config=ai_config, variation=variation + ) + pinned_tool_versions: dict[str, int] = {} + if ai_config is not None and variation is not None: + ai_config_variation = await asyncio.to_thread( + self._runner._fetch_config_variation, project_key, ai_config, variation + ) + generation = _merge_generation(ai_config_variation.generation, generation) + if tools is None and ai_config_variation.tool_versions: + raise EvaluationsError( + f"AI Config variation {ai_config!r}/{variation!r} uses tools " + "with no implementation: " + + ", ".join( + repr(name) for name in ai_config_variation.tool_versions + ) + + ". Pass tools= with an implementation for each." + ) + pinned_tool_versions = ai_config_variation.tool_versions + if criteria is None: + criteria = [ + Judge(key=judge_key) for judge_key in ai_config_variation.judge_keys + ] + generation = self._validate_generation(generation) run_tools = dict(tools or {}) run_criteria = list(criteria or []) run_judge_handlers = list(judge_handlers or []) @@ -157,6 +220,20 @@ async def run( resolved_tools = await asyncio.to_thread( self._runner._resolve_tools, project_key, run_tools ) + # The tool API serves only the latest version, so a variation pinned to + # an older one is evaluated against the current schema. + for tool_key, pinned_version in pinned_tool_versions.items(): + resolved_tool = resolved_tools.get(tool_key) + if resolved_tool is not None and resolved_tool.version != pinned_version: + logger.warning( + "AI Config variation %r/%r pins tool %r at version %d; " + "evaluating against the latest version %d.", + ai_config, + variation, + tool_key, + pinned_version, + resolved_tool.version, + ) resolved_judges = await self._runner._resolve_judges( project_key, ld_judges, handler, run_judge_handlers ) @@ -355,7 +432,6 @@ def _validate_run_args( key: str, dataset: str, handler: EvalHandler, - generation: GenerationConfig, concurrency: int, poll_interval_seconds: float, poll_timeout_seconds: float, @@ -369,16 +445,6 @@ def _validate_run_args( raise EvaluationsError(f"{name} must not be blank") if not callable(handler): raise EvaluationsError("handler must be callable") - provider = generation.get("provider") - model = generation.get("model") - if not isinstance(provider, str) or not provider.strip(): - raise EvaluationsError("generation.provider is required") - if not isinstance(model, str) or not model.strip(): - raise EvaluationsError("generation.model is required") - if "instructions" in generation and "messages" in generation: - raise EvaluationsError( - "generation.instructions and generation.messages are mutually exclusive" - ) if concurrency < 1: raise EvaluationsError("concurrency must be at least 1") for name, seconds in ( @@ -391,6 +457,49 @@ def _validate_run_args( if seconds < 0: raise EvaluationsError(f"{name} must not be negative") + @staticmethod + def _validate_config_source( + *, + generation: GenerationConfig | None, + ai_config: str | None, + variation: str | None, + ) -> None: + """Require a generation source before any request is made.""" + if ai_config is None and variation is None: + if generation is None: + raise EvaluationsError( + "Pass generation, or ai_config and variation to evaluate an " + "existing AI Config variation" + ) + return + if ai_config is None: + raise EvaluationsError("variation requires ai_config") + if variation is None: + raise EvaluationsError("ai_config requires variation") + for name, value in (("ai_config", ai_config), ("variation", variation)): + if not value.strip(): + raise EvaluationsError(f"{name} must not be blank") + + @staticmethod + def _validate_generation(generation: GenerationConfig | None) -> GenerationConfig: + """Check the final generation settings, after any fetched variation is merged.""" + if generation is None: + raise EvaluationsError( + "Pass generation, or ai_config and variation to evaluate an " + "existing AI Config variation" + ) + provider = generation.get("provider") + model = generation.get("model") + if not isinstance(provider, str) or not provider.strip(): + raise EvaluationsError("generation.provider is required") + if not isinstance(model, str) or not model.strip(): + raise EvaluationsError("generation.model is required") + if "instructions" in generation and "messages" in generation: + raise EvaluationsError( + "generation.instructions and generation.messages are mutually exclusive" + ) + return generation + def init_evaluations( api_token: str | None = None, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 164ea57d..9f9cc642 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -36,6 +36,7 @@ TokenUsage, ) from .types import ( + AIConfigVariation, DatasetRef, DatasetRow, EvaluationRef, @@ -214,6 +215,134 @@ class EvaluationsRunner: def __init__(self, api: LDApiClient) -> None: self._api = api + def _fetch_config_variation( + self, + project_key: str, + config_key: str, + variation_key: str, + ) -> AIConfigVariation: + """Read an AI Config variation by key from the management API. + + Flag delivery cannot select a variation by key -- it serves whichever + variation targeting picks for a context -- so this reads the variation + definition directly. Provider and base model parameters live on the + linked model config, and are layered the way the served flag payload + layers them: model-config parameters first, variation parameters over. + """ + description = f"AI Config variation {config_key!r}/{variation_key!r}" + path = ( + f"projects/{_segment(project_key)}/ai-configs/{_segment(config_key)}" + f"/variations/{_segment(variation_key)}" + ) + try: + raw = _mapping(self._api.get(path), description=description) + except LDApiError as error: + if error.status == 404: + raise EvaluationsError( + f"LaunchDarkly {description} was not found in project {project_key!r}" + ) from error + raise + # The endpoint returns every version of the variation; evaluate the latest. + items = raw.get("items") + versions = [ + item + for item in (items if isinstance(items, list) else []) + if isinstance(item, Mapping) and isinstance(item.get("version"), int) + ] + if not versions: + raise EvaluationsError(f"LaunchDarkly {description} has no versions") + latest = max(versions, key=lambda item: int(item["version"])) + + model = latest.get("model") + model = model if isinstance(model, Mapping) else {} + model_name = model.get("modelName") + variation_parameters = model.get("parameters") + parameters: dict[str, Any] = dict( + variation_parameters if isinstance(variation_parameters, Mapping) else {} + ) + provider: Any = None + model_config_key = latest.get("modelConfigKey") + if isinstance(model_config_key, str) and model_config_key: + model_config = self._fetch_model_config( + project_key, model_config_key, latest.get("modelConfigVersion") + ) + provider = model_config.get("provider") + base_parameters = model_config.get("params") + if isinstance(base_parameters, Mapping): + parameters = {**base_parameters, **parameters} + if not model_name: + model_name = model_config.get("id") + + generation = GenerationConfig() + if isinstance(provider, str) and provider: + generation["provider"] = provider + if isinstance(model_name, str) and model_name: + generation["model"] = model_name + if parameters: + generation["parameters"] = parameters + instructions = latest.get("instructions") + messages = latest.get("messages") + if isinstance(instructions, str) and instructions: + generation["instructions"] = instructions + elif isinstance(messages, list) and messages: + generation["messages"] = [ + dict(message) for message in messages if isinstance(message, Mapping) + ] + output_format = latest.get("outputFormat") + if isinstance(output_format, Mapping): + generation["output_format"] = dict(output_format) + + tools = latest.get("tools") + tool_versions = { + tool["key"]: tool["version"] + for tool in (tools if isinstance(tools, list) else []) + if isinstance(tool, Mapping) + and isinstance(tool.get("key"), str) + and isinstance(tool.get("version"), int) + } + judge_configuration = latest.get("judgeConfiguration") + judges = ( + judge_configuration.get("judges") + if isinstance(judge_configuration, Mapping) + else None + ) + judge_keys = [ + judge["judgeConfigKey"] + for judge in (judges if isinstance(judges, list) else []) + if isinstance(judge, Mapping) + and isinstance(judge.get("judgeConfigKey"), str) + ] + return AIConfigVariation( + generation=generation, + tool_versions=tool_versions, + judge_keys=judge_keys, + ) + + def _fetch_model_config( + self, + project_key: str, + model_config_key: str, + version: Any, + ) -> Mapping[str, Any]: + path = ( + f"projects/{_segment(project_key)}/ai-configs/model-configs/" + f"{_segment(model_config_key)}" + ) + # A pinned variation names the model-config version it was built against. + params = {"version": version} if isinstance(version, int) else None + try: + return _mapping( + self._api.get(path, params=params), + description=f"model config {model_config_key!r}", + ) + except LDApiError as error: + if error.status == 404: + raise EvaluationsError( + f"LaunchDarkly model config {model_config_key!r} was not found " + f"in project {project_key!r}" + ) from error + raise + def _resolve_tools( self, project_key: str, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index f397859b..c6dfb089 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -67,6 +67,21 @@ class ResolvedTool: schema: dict[str, Any] = field(default_factory=dict) +@dataclass +class AIConfigVariation: + """An AI Config variation read from the management API as run() defaults. + + ``generation`` holds only the fields the variation defines, so a caller's + explicit arguments can be layered over it. ``tool_versions`` maps each + attached tool key to the version the variation pins; ``judge_keys`` lists + the judges attached to the variation. + """ + + generation: GenerationConfig + tool_versions: dict[str, int] = field(default_factory=dict) + judge_keys: list[str] = field(default_factory=list) + + @dataclass class ResolvedJudge: """A LaunchDarkly AI Judge config variation resolved for an evaluation run. diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index c7ca278b..e9a579a9 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2249,3 +2249,310 @@ async def handler( assert result.passed is True assert max_in_flight == 2 + + +def config_variation_page(**overrides: Any) -> dict[str, Any]: + """A getAIConfigVariation response holding two versions of one variation.""" + latest: dict[str, Any] = { + "_id": "variation-id", + "key": "control", + "name": "Control", + "version": 2, + "createdAt": 2, + "model": {"modelName": "gpt-4o", "parameters": {"temperature": 0.7}}, + "modelConfigKey": "OpenAI.gpt-4o", + "modelConfigVersion": 3, + "instructions": "You are a support agent.", + **overrides, + } + stale = { + **latest, + "version": 1, + "createdAt": 1, + "instructions": "stale prompt", + } + return {"items": [stale, latest], "totalCount": 2} + + +MODEL_CONFIG = { + "key": "OpenAI.gpt-4o", + "id": "gpt-4o", + "name": "GPT-4o", + "provider": "OpenAI", + "params": {"max_tokens": 100, "temperature": 1.0}, + "version": 3, +} + + +def fetched_run_responses(variation_page: dict[str, Any]) -> list[HttpResponse]: + """Every response a run seeded from an AI Config variation needs, in order.""" + return [ + response(200, variation_page), + response(200, MODEL_CONFIG), + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page([{"rowIndex": 0, "input": "hello", "variables": {}}], total=1), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + { + "statusCounts": { + "total": 1, + "passed": 1, + "failed": 0, + "error": 0, + "pending": 0, + } + }, + ), + ] + + +def evaluation_post(transport: SequencedTransport) -> dict[str, Any]: + posts = [ + request + for request in transport.requests + if request["method"] == "POST" and request["url"].endswith("/evaluations") + ] + assert len(posts) == 1 + body: dict[str, Any] = posts[0]["body"] + return body + + +@pytest.mark.asyncio +async def test_run_seeds_generation_from_the_latest_ai_config_variation() -> None: + transport = SequencedTransport(fetched_run_responses(config_variation_page())) + evals = init_evaluations(api_token="token", transport=transport) + seen_configs: list[dict[str, Any]] = [] + + async def handler(config: dict[str, Any], *args: object) -> dict[str, Any]: + seen_configs.append(config) + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + ai_config="support-agent", + variation="control", + ) + + assert result.passed is True + assert transport.requests[0]["url"] == ( + "https://app.launchdarkly.com/api/v2/projects/proj/ai-configs/" + "support-agent/variations/control" + ) + # The pinned model-config version is the one read. + assert transport.requests[1]["url"].endswith( + "/projects/proj/ai-configs/model-configs/OpenAI.gpt-4o?version=3" + ) + body = evaluation_post(transport) + assert body["generationProvider"] == "OpenAI" + assert body["generationModel"] == "gpt-4o" + # Model-config parameters sit under the variation's own, as flag delivery layers them. + assert body["parameters"] == {"max_tokens": 100, "temperature": 0.7} + assert body["messages"] == [ + {"role": "system", "content": "You are a support agent."} + ] + assert seen_configs[0]["provider"] == {"name": "OpenAI"} + assert seen_configs[0]["instructions"] == "You are a support agent." + + +@pytest.mark.asyncio +async def test_explicit_generation_overrides_the_fetched_variation() -> None: + transport = SequencedTransport(fetched_run_responses(config_variation_page())) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + ai_config="support-agent", + variation="control", + generation={ + "model": "gpt-4o-mini", + "parameters": {"temperature": 0.1}, + "messages": [{"role": "system", "content": "Candidate prompt"}], + }, + ) + + body = evaluation_post(transport) + assert body["generationProvider"] == "OpenAI" + assert body["generationModel"] == "gpt-4o-mini" + # parameters merge key by key rather than replacing the fetched set. + assert body["parameters"] == {"max_tokens": 100, "temperature": 0.1} + # messages replace the fetched instructions instead of clashing with them. + assert body["messages"] == [{"role": "system", "content": "Candidate prompt"}] + + +@pytest.mark.asyncio +async def test_variation_tools_without_implementations_fail_before_mutating_requests() -> ( + None +): + transport = SequencedTransport( + [ + response( + 200, + config_variation_page(tools=[{"key": "lookup_order", "version": 4}]), + ), + response(200, MODEL_CONFIG), + ] + ) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match="'lookup_order'"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + ai_config="support-agent", + variation="control", + ) + + assert [request["method"] for request in transport.requests] == ["GET", "GET"] + + +@pytest.mark.asyncio +async def test_variation_judges_become_the_default_criteria( + monkeypatch: pytest.MonkeyPatch, +) -> None: + transport = SequencedTransport( + [ + response( + 200, + config_variation_page( + judgeConfiguration={ + "judges": [ + {"judgeConfigKey": "security-judge", "samplingRate": 1.0} + ] + } + ), + ), + response(200, MODEL_CONFIG), + ] + ) + + async def fake_extract_variation( + key: str, context: dict[str, Any] + ) -> dict[str, Any]: + raise RuntimeError("not found") + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.runner.extract_variation", + fake_extract_variation, + ) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + # Resolving the attached judge is what fails, so it was picked up as a criterion. + with pytest.raises(EvaluationsError, match="'security-judge'"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + ai_config="support-agent", + variation="control", + ) + + assert [request["method"] for request in transport.requests] == ["GET", "GET"] + + +@pytest.mark.asyncio +async def test_unknown_variation_fails_before_any_records_are_created() -> None: + transport = SequencedTransport( + [response(404, {"code": "not_found", "message": "not found"})] + ) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match="was not found"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + ai_config="support-agent", + variation="missing", + ) + + assert [request["method"] for request in transport.requests] == ["GET"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("source", "message"), + [ + ({}, "Pass generation"), + ({"variation": "control"}, "variation requires ai_config"), + ({"ai_config": "support-agent"}, "ai_config requires variation"), + ({"ai_config": " ", "variation": "control"}, "ai_config must not be blank"), + ], +) +async def test_config_source_is_validated_before_network_io( + source: dict[str, str], message: str +) -> None: + transport = SequencedTransport([]) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match=message): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + **source, # type: ignore[arg-type] + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_tool_version_drift_from_the_variation_is_logged( + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level("WARNING", logger="launchdarkly_ai_server.evaluations.module") + responses = fetched_run_responses( + config_variation_page(tools=[{"key": "lookup_order", "version": 4}]) + ) + responses.insert( + 2, + response( + 200, + {"key": "lookup_order", "version": 7, "schema": {"type": "object"}}, + ), + ) + transport = SequencedTransport(responses) + evals = init_evaluations(api_token="token", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + ai_config="support-agent", + variation="control", + tools={"lookup_order": lookup_order}, + ) + + assert "pins tool 'lookup_order' at version 4" in caplog.text + assert "latest version 7" in caplog.text + assert evaluation_post(transport)["tools"] == [ + {"key": "lookup_order", "version": 7} + ] From 76d83dc04b839693240434bf85592c288c830b1b Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Thu, 24 Sep 2026 15:25:17 -0400 Subject: [PATCH 2/4] model config validation refactor --- .../evaluations/runner.py | 10 +++- packages/client/tests/test_evaluations_run.py | 47 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 9f9cc642..724f5393 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -262,7 +262,15 @@ def _fetch_config_variation( ) provider: Any = None model_config_key = latest.get("modelConfigKey") - if isinstance(model_config_key, str) and model_config_key: + # Absent or empty means the variation links no model config, so it has + # no provider -- flag delivery serves an empty provider name for it too. + # Anything other than a string is a response we do not understand. + if model_config_key: + if not isinstance(model_config_key, str): + raise EvaluationsError( + f"LaunchDarkly {description} has a non-string modelConfigKey: " + f"{model_config_key!r}" + ) model_config = self._fetch_model_config( project_key, model_config_key, latest.get("modelConfigVersion") ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index e9a579a9..c782786d 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2556,3 +2556,50 @@ async def handler(*args: object) -> dict[str, Any]: assert evaluation_post(transport)["tools"] == [ {"key": "lookup_order", "version": 7} ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_config_key", [42, ["OpenAI.gpt-4o"], {"key": "x"}]) +async def test_non_string_model_config_key_fails_loudly( + model_config_key: object, +) -> None: + transport = SequencedTransport( + [response(200, config_variation_page(modelConfigKey=model_config_key))] + ) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match="non-string modelConfigKey"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + ai_config="support-agent", + variation="control", + ) + + assert [request["method"] for request in transport.requests] == ["GET"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_config_key", [None, ""]) +async def test_variation_without_a_model_config_needs_an_explicit_provider( + model_config_key: str | None, +) -> None: + transport = SequencedTransport( + [response(200, config_variation_page(modelConfigKey=model_config_key))] + ) + evals = init_evaluations(api_token="token", transport=transport) + + # No model config is linked, so none is fetched and no provider is known. + with pytest.raises(EvaluationsError, match=r"generation\.provider is required"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + ai_config="support-agent", + variation="control", + ) + + assert [request["method"] for request in transport.requests] == ["GET"] From 39a7e2e37317f7f2425ba82bca7043b9b0c8c36d Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Thu, 24 Sep 2026 15:59:16 -0400 Subject: [PATCH 3/4] move config variation into class method --- .../evaluations/runner.py | 65 +--------------- .../evaluations/types.py | 75 +++++++++++++++++++ packages/client/tests/test_evaluations_run.py | 32 ++++++-- 3 files changed, 106 insertions(+), 66 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 724f5393..4d4f9d00 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -253,19 +253,12 @@ def _fetch_config_variation( raise EvaluationsError(f"LaunchDarkly {description} has no versions") latest = max(versions, key=lambda item: int(item["version"])) - model = latest.get("model") - model = model if isinstance(model, Mapping) else {} - model_name = model.get("modelName") - variation_parameters = model.get("parameters") - parameters: dict[str, Any] = dict( - variation_parameters if isinstance(variation_parameters, Mapping) else {} - ) - provider: Any = None - model_config_key = latest.get("modelConfigKey") # Absent or empty means the variation links no model config, so it has # no provider -- flag delivery serves an empty provider name for it too. # Anything other than a string is a response we do not understand. - if model_config_key: + model_config: Mapping[str, Any] | None = None + model_config_key = latest.get("modelConfigKey") + if model_config_key is not None: if not isinstance(model_config_key, str): raise EvaluationsError( f"LaunchDarkly {description} has a non-string modelConfigKey: " @@ -274,57 +267,7 @@ def _fetch_config_variation( model_config = self._fetch_model_config( project_key, model_config_key, latest.get("modelConfigVersion") ) - provider = model_config.get("provider") - base_parameters = model_config.get("params") - if isinstance(base_parameters, Mapping): - parameters = {**base_parameters, **parameters} - if not model_name: - model_name = model_config.get("id") - - generation = GenerationConfig() - if isinstance(provider, str) and provider: - generation["provider"] = provider - if isinstance(model_name, str) and model_name: - generation["model"] = model_name - if parameters: - generation["parameters"] = parameters - instructions = latest.get("instructions") - messages = latest.get("messages") - if isinstance(instructions, str) and instructions: - generation["instructions"] = instructions - elif isinstance(messages, list) and messages: - generation["messages"] = [ - dict(message) for message in messages if isinstance(message, Mapping) - ] - output_format = latest.get("outputFormat") - if isinstance(output_format, Mapping): - generation["output_format"] = dict(output_format) - - tools = latest.get("tools") - tool_versions = { - tool["key"]: tool["version"] - for tool in (tools if isinstance(tools, list) else []) - if isinstance(tool, Mapping) - and isinstance(tool.get("key"), str) - and isinstance(tool.get("version"), int) - } - judge_configuration = latest.get("judgeConfiguration") - judges = ( - judge_configuration.get("judges") - if isinstance(judge_configuration, Mapping) - else None - ) - judge_keys = [ - judge["judgeConfigKey"] - for judge in (judges if isinstance(judges, list) else []) - if isinstance(judge, Mapping) - and isinstance(judge.get("judgeConfigKey"), str) - ] - return AIConfigVariation( - generation=generation, - tool_versions=tool_versions, - judge_keys=judge_keys, - ) + return AIConfigVariation.from_api(latest, model_config) def _fetch_model_config( self, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index c6dfb089..0eaf78c3 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -81,6 +81,81 @@ class AIConfigVariation: tool_versions: dict[str, int] = field(default_factory=dict) judge_keys: list[str] = field(default_factory=list) + @classmethod + def from_api( + cls, + data: Mapping[str, Any], + model_config: Mapping[str, Any] | None = None, + ) -> AIConfigVariation: + """Build from one variation version and the model config it links. + + ``model_config`` is the linked model-config response, or ``None`` when + the variation links none. Fetching it is the caller's job, so this stays + a pure translation of API shapes. Provider and base parameters come from + the model config; the variation's own parameters are layered over them, + as the served flag payload layers them. + """ + model = data.get("model") + model = model if isinstance(model, Mapping) else {} + model_name = model.get("modelName") + variation_parameters = model.get("parameters") + parameters: dict[str, Any] = dict( + variation_parameters if isinstance(variation_parameters, Mapping) else {} + ) + provider: Any = None + if model_config is not None: + provider = model_config.get("provider") + base_parameters = model_config.get("params") + if isinstance(base_parameters, Mapping): + parameters = {**base_parameters, **parameters} + if not model_name: + model_name = model_config.get("id") + + generation = GenerationConfig() + if isinstance(provider, str) and provider: + generation["provider"] = provider + if isinstance(model_name, str) and model_name: + generation["model"] = model_name + if parameters: + generation["parameters"] = parameters + instructions = data.get("instructions") + messages = data.get("messages") + if isinstance(instructions, str) and instructions: + generation["instructions"] = instructions + elif isinstance(messages, list) and messages: + generation["messages"] = [ + dict(message) for message in messages if isinstance(message, Mapping) + ] + output_format = data.get("outputFormat") + if isinstance(output_format, Mapping): + generation["output_format"] = dict(output_format) + + tools = data.get("tools") + tool_versions = { + tool["key"]: tool["version"] + for tool in (tools if isinstance(tools, list) else []) + if isinstance(tool, Mapping) + and isinstance(tool.get("key"), str) + and isinstance(tool.get("version"), int) + } + judge_configuration = data.get("judgeConfiguration") + judges = ( + judge_configuration.get("judges") + if isinstance(judge_configuration, Mapping) + else None + ) + judge_keys = [ + judge["judgeConfigKey"] + for judge in (judges if isinstance(judges, list) else []) + if isinstance(judge, Mapping) + and isinstance(judge.get("judgeConfigKey"), str) + ] + return cls( + generation=generation, + tool_versions=tool_versions, + judge_keys=judge_keys, + ) + @dataclass class ResolvedJudge: diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index c782786d..5d0ed569 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2582,12 +2582,9 @@ async def test_non_string_model_config_key_fails_loudly( @pytest.mark.asyncio -@pytest.mark.parametrize("model_config_key", [None, ""]) -async def test_variation_without_a_model_config_needs_an_explicit_provider( - model_config_key: str | None, -) -> None: +async def test_variation_without_a_model_config_needs_an_explicit_provider() -> None: transport = SequencedTransport( - [response(200, config_variation_page(modelConfigKey=model_config_key))] + [response(200, config_variation_page(modelConfigKey=None))] ) evals = init_evaluations(api_token="token", transport=transport) @@ -2603,3 +2600,28 @@ async def test_variation_without_a_model_config_needs_an_explicit_provider( ) assert [request["method"] for request in transport.requests] == ["GET"] + + +def test_ai_config_variation_from_api_layers_the_model_config() -> None: + from launchdarkly_ai_server.evaluations.types import AIConfigVariation + + latest = config_variation_page( + tools=[{"key": "lookup_order", "version": 4}], + judgeConfiguration={ + "judges": [{"judgeConfigKey": "security-judge", "samplingRate": 1.0}] + }, + )["items"][1] + + linked = AIConfigVariation.from_api(latest, MODEL_CONFIG) + assert linked.generation == { + "provider": "OpenAI", + "model": "gpt-4o", + "parameters": {"max_tokens": 100, "temperature": 0.7}, + "instructions": "You are a support agent.", + } + assert linked.tool_versions == {"lookup_order": 4} + assert linked.judge_keys == ["security-judge"] + + unlinked = AIConfigVariation.from_api(latest) + assert "provider" not in unlinked.generation + assert unlinked.generation["parameters"] == {"temperature": 0.7} From 510d21d810b104bbe1137f46292f69cce52d2e22 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Thu, 24 Sep 2026 16:11:07 -0400 Subject: [PATCH 4/4] don't fetch empty modelconfigkey --- .../src/launchdarkly_ai_server/evaluations/runner.py | 12 ++++++------ packages/client/tests/test_evaluations_run.py | 7 +++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 4d4f9d00..1b2a1db1 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -258,12 +258,12 @@ def _fetch_config_variation( # Anything other than a string is a response we do not understand. model_config: Mapping[str, Any] | None = None model_config_key = latest.get("modelConfigKey") - if model_config_key is not None: - if not isinstance(model_config_key, str): - raise EvaluationsError( - f"LaunchDarkly {description} has a non-string modelConfigKey: " - f"{model_config_key!r}" - ) + if model_config_key is not None and not isinstance(model_config_key, str): + raise EvaluationsError( + f"LaunchDarkly {description} has a non-string modelConfigKey: " + f"{model_config_key!r}" + ) + if model_config_key: model_config = self._fetch_model_config( project_key, model_config_key, latest.get("modelConfigVersion") ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 5d0ed569..0752febd 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2582,9 +2582,12 @@ async def test_non_string_model_config_key_fails_loudly( @pytest.mark.asyncio -async def test_variation_without_a_model_config_needs_an_explicit_provider() -> None: +@pytest.mark.parametrize("model_config_key", [None, ""]) +async def test_variation_without_a_model_config_needs_an_explicit_provider( + model_config_key: str | None, +) -> None: transport = SequencedTransport( - [response(200, config_variation_page(modelConfigKey=None))] + [response(200, config_variation_page(modelConfigKey=model_config_key))] ) evals = init_evaluations(api_token="token", transport=transport)