feat(AIC-3449): Support pulling in configs for evals from code

packages/client/src/launchdarkly_ai_server/evaluations/module.py

@@ -7,7 +7,7 @@
 import os
 import time
 from collections.abc import Mapping
-from typing import Any
+from typing import Any, cast
 
 from ..lifecycle import get_client, init_client
 from .api import (
@@ -63,6 +63,35 @@ def _is_terminal_summary(summary: RunSummary) -> bool:
     )
 
 
+def _merge_generation(
+    base: GenerationConfig, override: GenerationConfig | None
+) -> GenerationConfig:
+    """Layer a caller's generation settings over a fetched variation's.
+
+    Keys the caller sets replace the fetched ones, except ``parameters``, which
+    merge key by key so overriding ``temperature`` keeps a fetched
+    ``max_tokens``. ``instructions`` and ``messages`` are one prompt slot:
+    supplying either discards both fetched values, so a caller swapping an
+    agent prompt for a message list does not trip the mutual-exclusion check.
+    """
+    merged: dict[str, Any] = dict(base)
+    if not override:
+        return cast(GenerationConfig, merged)
+    if "instructions" in override or "messages" in override:
+        merged.pop("instructions", None)
+        merged.pop("messages", None)
+    for field_name, value in override.items():
+        if field_name == "parameters" and isinstance(value, Mapping):
+            base_parameters = merged.get("parameters")
+            merged["parameters"] = {
+                **(base_parameters if isinstance(base_parameters, Mapping) else {}),
+                **value,
+            }
+        else:
+            merged[field_name] = value
+    return cast(GenerationConfig, merged)
+
+
 class EvaluationsModule:
     """Entry point for running LaunchDarkly evaluations from customer code."""
 
@@ -98,7 +127,9 @@ async def run(
         key: str,
         dataset: str,
         handler: EvalHandler,
-        generation: GenerationConfig,
+        generation: GenerationConfig | None = None,
+        ai_config: str | None = None,
+        variation: str | None = None,
         tools: Mapping[str, ToolImplementation] | None = None,
         criteria: list[Criterion] | None = None,
         judge_handlers: list[EvalHandler] | None = None,
@@ -126,6 +157,15 @@ async def run(
         this method. Large datasets may need a longer ``poll_timeout_seconds``
         and a wider ``poll_interval_seconds``; both default to
         ``SUMMARY_POLL_TIMEOUT_SECONDS`` / ``SUMMARY_POLL_INTERVAL_SECONDS``.
+
+        Pass ``ai_config`` and ``variation`` to start from an existing AI Config
+        variation instead of a hand-built ``generation``. Its model, provider,
+        parameters, prompt and output format become the defaults, and anything
+        set in ``generation`` overrides them field by field (``parameters``
+        merge key by key). When ``tools`` is omitted the variation's tools are
+        used, so each needs an implementation; pass ``tools`` to replace the
+        set. When ``criteria`` is omitted the variation's attached judges run;
+        pass ``criteria`` (even ``[]``) to replace them.
         """
         if poll_interval_seconds is None:
             poll_interval_seconds = SUMMARY_POLL_INTERVAL_SECONDS
@@ -136,11 +176,34 @@ async def run(
             key=key,
             dataset=dataset,
             handler=handler,
-            generation=generation,
             concurrency=concurrency,
             poll_interval_seconds=poll_interval_seconds,
             poll_timeout_seconds=poll_timeout_seconds,
         )
+        self._validate_config_source(
+            generation=generation, ai_config=ai_config, variation=variation
+        )
+        pinned_tool_versions: dict[str, int] = {}
+        if ai_config is not None and variation is not None:
+            ai_config_variation = await asyncio.to_thread(
+                self._runner._fetch_config_variation, project_key, ai_config, variation
+            )
+            generation = _merge_generation(ai_config_variation.generation, generation)
+            if tools is None and ai_config_variation.tool_versions:
+                raise EvaluationsError(
+                    f"AI Config variation {ai_config!r}/{variation!r} uses tools "
+                    "with no implementation: "
+                    + ", ".join(
+                        repr(name) for name in ai_config_variation.tool_versions
+                    )
+                    + ". Pass tools= with an implementation for each."
+                )
+            pinned_tool_versions = ai_config_variation.tool_versions
+            if criteria is None:
+                criteria = [
+                    Judge(key=judge_key) for judge_key in ai_config_variation.judge_keys
+                ]
+        generation = self._validate_generation(generation)
         run_tools = dict(tools or {})
         run_criteria = list(criteria or [])
         run_judge_handlers = list(judge_handlers or [])
@@ -157,6 +220,20 @@ async def run(
         resolved_tools = await asyncio.to_thread(
             self._runner._resolve_tools, project_key, run_tools
         )
+        # The tool API serves only the latest version, so a variation pinned to
+        # an older one is evaluated against the current schema.
+        for tool_key, pinned_version in pinned_tool_versions.items():
+            resolved_tool = resolved_tools.get(tool_key)
+            if resolved_tool is not None and resolved_tool.version != pinned_version:
+                logger.warning(
+                    "AI Config variation %r/%r pins tool %r at version %d; "
+                    "evaluating against the latest version %d.",
+                    ai_config,
+                    variation,
+                    tool_key,
+                    pinned_version,
+                    resolved_tool.version,
+                )
         resolved_judges = await self._runner._resolve_judges(
             project_key, ld_judges, handler, run_judge_handlers
         )
@@ -355,7 +432,6 @@ def _validate_run_args(
         key: str,
         dataset: str,
         handler: EvalHandler,
-        generation: GenerationConfig,
         concurrency: int,
         poll_interval_seconds: float,
         poll_timeout_seconds: float,
@@ -369,16 +445,6 @@ def _validate_run_args(
                 raise EvaluationsError(f"{name} must not be blank")
         if not callable(handler):
             raise EvaluationsError("handler must be callable")
-        provider = generation.get("provider")
-        model = generation.get("model")
-        if not isinstance(provider, str) or not provider.strip():
-            raise EvaluationsError("generation.provider is required")
-        if not isinstance(model, str) or not model.strip():
-            raise EvaluationsError("generation.model is required")
-        if "instructions" in generation and "messages" in generation:
-            raise EvaluationsError(
-                "generation.instructions and generation.messages are mutually exclusive"
-            )
         if concurrency < 1:
             raise EvaluationsError("concurrency must be at least 1")
         for name, seconds in (
@@ -391,6 +457,49 @@ def _validate_run_args(
             if seconds < 0:
                 raise EvaluationsError(f"{name} must not be negative")
 
+    @staticmethod
+    def _validate_config_source(
+        *,
+        generation: GenerationConfig | None,
+        ai_config: str | None,
+        variation: str | None,
+    ) -> None:
+        """Require a generation source before any request is made."""
+        if ai_config is None and variation is None:
+            if generation is None:
+                raise EvaluationsError(
+                    "Pass generation, or ai_config and variation to evaluate an "
+                    "existing AI Config variation"
+                )
+            return
+        if ai_config is None:
+            raise EvaluationsError("variation requires ai_config")
+        if variation is None:
+            raise EvaluationsError("ai_config requires variation")
+        for name, value in (("ai_config", ai_config), ("variation", variation)):
+            if not value.strip():
+                raise EvaluationsError(f"{name} must not be blank")
+
+    @staticmethod
+    def _validate_generation(generation: GenerationConfig | None) -> GenerationConfig:
+        """Check the final generation settings, after any fetched variation is merged."""
+        if generation is None:
+            raise EvaluationsError(
+                "Pass generation, or ai_config and variation to evaluate an "
+                "existing AI Config variation"
+            )
+        provider = generation.get("provider")
+        model = generation.get("model")
+        if not isinstance(provider, str) or not provider.strip():
+            raise EvaluationsError("generation.provider is required")
+        if not isinstance(model, str) or not model.strip():
+            raise EvaluationsError("generation.model is required")
+        if "instructions" in generation and "messages" in generation:
+            raise EvaluationsError(
+                "generation.instructions and generation.messages are mutually exclusive"
+            )
+        return generation
+
 
 def init_evaluations(
     api_token: str | None = None,

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -36,6 +36,7 @@
     TokenUsage,
 )
 from .types import (
+    AIConfigVariation,
     DatasetRef,
     DatasetRow,
     EvaluationRef,
@@ -214,6 +215,85 @@ class EvaluationsRunner:
     def __init__(self, api: LDApiClient) -> None:
         self._api = api
 
+    def _fetch_config_variation(
+        self,
+        project_key: str,
+        config_key: str,
+        variation_key: str,
+    ) -> AIConfigVariation:
+        """Read an AI Config variation by key from the management API.
+
+        Flag delivery cannot select a variation by key -- it serves whichever
+        variation targeting picks for a context -- so this reads the variation
+        definition directly. Provider and base model parameters live on the
+        linked model config, and are layered the way the served flag payload
+        layers them: model-config parameters first, variation parameters over.
+        """
+        description = f"AI Config variation {config_key!r}/{variation_key!r}"
+        path = (
+            f"projects/{_segment(project_key)}/ai-configs/{_segment(config_key)}"
+            f"/variations/{_segment(variation_key)}"
+        )
+        try:
+            raw = _mapping(self._api.get(path), description=description)
+        except LDApiError as error:
+            if error.status == 404:
+                raise EvaluationsError(
+                    f"LaunchDarkly {description} was not found in project {project_key!r}"
+                ) from error
+            raise
+        # The endpoint returns every version of the variation; evaluate the latest.
+        items = raw.get("items")
+        versions = [
+            item
+            for item in (items if isinstance(items, list) else [])
+            if isinstance(item, Mapping) and isinstance(item.get("version"), int)
+        ]
+        if not versions:
+            raise EvaluationsError(f"LaunchDarkly {description} has no versions")
+        latest = max(versions, key=lambda item: int(item["version"]))
+
+        # Absent or empty means the variation links no model config, so it has
+        # no provider -- flag delivery serves an empty provider name for it too.
+        # Anything other than a string is a response we do not understand.
+        model_config: Mapping[str, Any] | None = None
+        model_config_key = latest.get("modelConfigKey")
+        if model_config_key is not None and not isinstance(model_config_key, str):
+            raise EvaluationsError(
+                f"LaunchDarkly {description} has a non-string modelConfigKey: "
+                f"{model_config_key!r}"
+            )
+        if model_config_key:
+            model_config = self._fetch_model_config(
+                project_key, model_config_key, latest.get("modelConfigVersion")
+            )
+        return AIConfigVariation.from_api(latest, model_config)
+
+    def _fetch_model_config(
+        self,
+        project_key: str,
+        model_config_key: str,
+        version: Any,
+    ) -> Mapping[str, Any]:
+        path = (
+            f"projects/{_segment(project_key)}/ai-configs/model-configs/"
+            f"{_segment(model_config_key)}"
+        )
+        # A pinned variation names the model-config version it was built against.
+        params = {"version": version} if isinstance(version, int) else None
+        try:
+            return _mapping(
+                self._api.get(path, params=params),
+                description=f"model config {model_config_key!r}",
+            )
+        except LDApiError as error:
+            if error.status == 404:
+                raise EvaluationsError(
+                    f"LaunchDarkly model config {model_config_key!r} was not found "
+                    f"in project {project_key!r}"
+                ) from error
+            raise
+
     def _resolve_tools(
         self,
         project_key: str,

packages/client/src/launchdarkly_ai_server/evaluations/types.py

@@ -67,6 +67,96 @@ class ResolvedTool:
     schema: dict[str, Any] = field(default_factory=dict)
 
 
+@dataclass
+class AIConfigVariation:
+    """An AI Config variation read from the management API as run() defaults.
+
+    ``generation`` holds only the fields the variation defines, so a caller's
+    explicit arguments can be layered over it. ``tool_versions`` maps each
+    attached tool key to the version the variation pins; ``judge_keys`` lists
+    the judges attached to the variation.
+    """
+
+    generation: GenerationConfig
+    tool_versions: dict[str, int] = field(default_factory=dict)
+    judge_keys: list[str] = field(default_factory=list)
+
+    @classmethod
+    def from_api(
+        cls,
+        data: Mapping[str, Any],
+        model_config: Mapping[str, Any] | None = None,
+    ) -> AIConfigVariation:
+        """Build from one variation version and the model config it links.
+
+        ``model_config`` is the linked model-config response, or ``None`` when
+        the variation links none. Fetching it is the caller's job, so this stays
+        a pure translation of API shapes. Provider and base parameters come from
+        the model config; the variation's own parameters are layered over them,
+        as the served flag payload layers them.
+        """
+        model = data.get("model")
+        model = model if isinstance(model, Mapping) else {}
+        model_name = model.get("modelName")
+        variation_parameters = model.get("parameters")
+        parameters: dict[str, Any] = dict(
+            variation_parameters if isinstance(variation_parameters, Mapping) else {}
+        )
+        provider: Any = None
+        if model_config is not None:
+            provider = model_config.get("provider")
+            base_parameters = model_config.get("params")
+            if isinstance(base_parameters, Mapping):
+                parameters = {**base_parameters, **parameters}
+            if not model_name:
+                model_name = model_config.get("id")
+
+        generation = GenerationConfig()
+        if isinstance(provider, str) and provider:
+            generation["provider"] = provider
+        if isinstance(model_name, str) and model_name:
+            generation["model"] = model_name
+        if parameters:
+            generation["parameters"] = parameters
+        instructions = data.get("instructions")
+        messages = data.get("messages")
+        if isinstance(instructions, str) and instructions:
+            generation["instructions"] = instructions
+        elif isinstance(messages, list) and messages:
+            generation["messages"] = [
+                dict(message) for message in messages if isinstance(message, Mapping)
+            ]
+        output_format = data.get("outputFormat")
+        if isinstance(output_format, Mapping):
+            generation["output_format"] = dict(output_format)
+
+        tools = data.get("tools")
+        tool_versions = {
+            tool["key"]: tool["version"]
+            for tool in (tools if isinstance(tools, list) else [])
+            if isinstance(tool, Mapping)
+            and isinstance(tool.get("key"), str)
+            and isinstance(tool.get("version"), int)
+        }
+        judge_configuration = data.get("judgeConfiguration")
+        judges = (
+            judge_configuration.get("judges")
+            if isinstance(judge_configuration, Mapping)
+            else None
+        )
+        judge_keys = [
+            judge["judgeConfigKey"]
+            for judge in (judges if isinstance(judges, list) else [])
+            if isinstance(judge, Mapping)
+            and isinstance(judge.get("judgeConfigKey"), str)
+        ]
+        return cls(
+            generation=generation,
+            tool_versions=tool_versions,
+            judge_keys=judge_keys,
+        )
+
+
 @dataclass
 class ResolvedJudge:
     """A LaunchDarkly AI Judge config variation resolved for an evaluation run.