feat: Support creating evaluation runs from pre-existing Interactions API data

agentplatform/_genai/_evals_common.py

@@ -400,24 +400,187 @@ def _is_n_plus_1_inference(
 def _extract_response_from_completed_trace(
     agent_data: types.evals.AgentData,
 ) -> list[dict[str, Any]]:
-    """Extracts all events from a completed agent trace as event dicts.
+  """Extracts all events from a completed agent trace as event dicts.
 
     For BYOD (bring-your-own-data) use cases where the agent trace is
     already complete, this returns all events formatted as a list of
     dicts compatible with ``_process_single_turn_agent_response``.  The
     last element is the final agent response; preceding elements become
     intermediate events.
     """
-    event_dicts: list[dict[str, Any]] = []
-    for turn in agent_data.turns or []:
-        if not turn.events:
-            continue
-        for event in turn.events:
-            d: dict[str, Any] = {"author": event.author or "agent"}
-            if event.content:
-                d[CONTENT] = event.content.model_dump(exclude_none=True)
-            event_dicts.append(d)
-    return event_dicts
+  event_dicts: list[dict[str, Any]] = []
+  for turn in agent_data.turns or []:
+    if not turn.events:
+      continue
+    for event in turn.events:
+      d: dict[str, Any] = {"author": event.author or "agent"}
+      if event.content:
+        d[CONTENT] = event.content.model_dump(exclude_none=True)
+      event_dicts.append(d)
+  return event_dicts
+
+
+def _has_interactions_data_source(
+    eval_cases: list[types.EvalCase],
+) -> bool:
+  """Returns True if any EvalCase has interactions_data_source set."""
+  return any(
+      getattr(case, "interactions_data_source", None) is not None
+      for case in eval_cases
+  )
+
+
+def _interaction_dict_to_agent_data(
+    interaction: dict[str, Any],
+) -> dict[str, Any]:
+  """Converts an Interaction API JSON response to an AgentData-compatible dict.
+
+  Maps the flat list of Interaction steps (user_input, model_output,
+  function_call, function_result, etc.) into a single ConversationTurn
+  with AgentEvents, matching the AgentData structure expected by the
+  evaluation pipeline.
+
+  Args:
+      interaction: A dict from the Interactions API GET response.
+
+  Returns:
+      A dict matching the AgentData schema with a single turn containing
+      all events from the interaction.
+  """
+  events = []
+  for step in interaction.get("steps", []):
+    step_type = step.get("type")
+
+    if step_type == "user_input":
+      parts = []
+      for content_item in step.get("content", []):
+        if content_item.get("type") == "text":
+          parts.append({"text": content_item.get("text", "")})
+      if parts:
+        events.append({
+            "author": "user",
+            "content": {"role": "user", "parts": parts},
+        })
+
+    elif step_type == "model_output":
+      parts = []
+      for content_item in step.get("content", []):
+        if content_item.get("type") == "text":
+          parts.append({"text": content_item.get("text", "")})
+      if parts:
+        events.append({
+            "author": "agent",
+            "content": {"role": "model", "parts": parts},
+        })
+
+    elif step_type == "function_call":
+      events.append({
+          "author": "agent",
+          "content": {
+              "role": "model",
+              "parts": [{
+                  "function_call": {
+                      "name": step.get("name", ""),
+                      "args": step.get("arguments", {}),
+                      "id": step.get("id", ""),
+                  },
+              }],
+          },
+      })
+
+    elif step_type == "function_result":
+      result = step.get("result")
+      if isinstance(result, dict):
+        result_str = json.dumps(result)
+      elif isinstance(result, str):
+        result_str = result
+      else:
+        result_str = str(result) if result is not None else ""
+      events.append({
+          "author": "user",
+          "content": {
+              "role": "user",
+              "parts": [{
+                  "function_response": {
+                      "name": step.get("name", ""),
+                      "response": {"result": result_str},
+                      "id": step.get("callId", step.get("call_id", "")),
+                  },
+              }],
+          },
+      })
+
+  agent_data: dict[str, Any] = {
+      "turns": [{
+          "turn_index": 0,
+          "events": events,
+      }],
+  }
+  return agent_data
+
+
+def _resolve_interactions_to_eval_cases(
+    api_client: BaseApiClient,
+    eval_cases: list[types.EvalCase],
+) -> list[types.EvalCase]:
+  """Resolves EvalCases with interactions_data_source to agent_data.
+
+  For each EvalCase that has interactions_data_source set, fetches the
+  Interaction via the SDK's interactions.get() API, converts the steps
+  to AgentData, and returns a new EvalCase with agent_data populated.
+
+  Args:
+      api_client: The API client (must have an interactions module).
+      eval_cases: EvalCases with interactions_data_source set.
+
+  Returns:
+      New list of EvalCases with agent_data populated from resolved
+      interactions.
+
+  Raises:
+      ValueError: If eval_cases have missing interaction references.
+  """
+  # Validate all cases up front before making any API calls.
+  for case in eval_cases:
+    ids = case.interactions_data_source
+    if ids is None:
+      raise ValueError(
+          "All eval_cases must have interactions_data_source set when"
+          " using interaction resolution. Found a case without it. Do"
+          " not mix interaction-based and prompt-based eval cases."
+      )
+    if not ids.interaction:
+      raise ValueError(
+          "interactions_data_source.interaction is required. Each"
+          " EvalCase must reference an existing Interaction resource."
+      )
+
+  resolved_cases = []
+
+  for case in eval_cases:
+    ids = case.interactions_data_source
+
+    # Extract the interaction ID from the resource name.
+    # Format: projects/{p}/locations/{l}/interactions/{id}
+    parts = ids.interaction.split("/")
+    if len(parts) >= 6 and parts[4] == "interactions":
+      interaction_id = parts[5]
+    else:
+      interaction_id = ids.interaction
+
+    logger.info("Fetching interaction: %s", ids.interaction)
+    path = f"interactions/{interaction_id}"
+    response = api_client.request("get", path, {}, None)
+    interaction_dict = (
+        {} if not response.body else json.loads(response.body)
+    )
+
+    agent_data = _interaction_dict_to_agent_data(interaction_dict)
+    resolved_cases.append(
+        types.EvalCase(agent_data=agent_data)
+    )
+
+  return resolved_cases
 
 
 def _resolve_dataset(
@@ -426,22 +589,31 @@ def _resolve_dataset(
     dest: str,
     parsed_agent_info: Optional[types.evals.AgentInfo] = None,
 ) -> types.EvaluationRunDataSource:
-    """Resolves dataset for the evaluation run."""
-    if isinstance(dataset, types.EvaluationDataset):
-        candidate_name = _get_candidate_name(dataset, parsed_agent_info)
-        eval_df = dataset.eval_dataset_df
-        if eval_df is None and dataset.eval_cases:
-            eval_df = _eval_cases_to_dataframe(dataset.eval_cases)
-
-        eval_set = _create_evaluation_set_from_dataframe(
-            api_client,
-            dest,
-            eval_df,
-            candidate_name,
-            parsed_agent_info=parsed_agent_info,
-        )
-        dataset = types.EvaluationRunDataSource(evaluation_set=eval_set.name)
-    return dataset
+  """Resolves dataset for the evaluation run."""
+  if isinstance(dataset, types.EvaluationDataset):
+    # Resolve EvalCases with interactions_data_source by fetching
+    # each interaction and converting it to agent_data, then flowing
+    # through the normal DataFrame/GCS pipeline.
+    if dataset.eval_cases and _has_interactions_data_source(dataset.eval_cases):
+      resolved_cases = _resolve_interactions_to_eval_cases(
+          api_client, dataset.eval_cases
+      )
+      dataset = types.EvaluationDataset(eval_cases=resolved_cases)
+
+    candidate_name = _get_candidate_name(dataset, parsed_agent_info)
+    eval_df = dataset.eval_dataset_df
+    if eval_df is None and dataset.eval_cases:
+      eval_df = _eval_cases_to_dataframe(dataset.eval_cases)
+
+    eval_set = _create_evaluation_set_from_dataframe(
+        api_client,
+        dest,
+        eval_df,
+        candidate_name,
+        parsed_agent_info=parsed_agent_info,
+    )
+    dataset = types.EvaluationRunDataSource(evaluation_set=eval_set.name)
+  return dataset
 
 
 def _get_default_prompt_template(

tests/unit/agentplatform/genai/test_evals.py

@@ -9563,6 +9563,139 @@ def test_resolve_dataset_preserves_conversation_history(
         assert "conversation_history" in ptd_values
 
 
+class TestResolveDatasetWithInteractions:
+    """Tests for resolving interactions_data_source in _resolve_dataset."""
+
+    def setup_method(self):
+        self.mock_api_client = mock.Mock()
+        self.mock_api_client.project = "test-project"
+        self.mock_api_client.location = "us-central1"
+
+    def test_has_interactions_data_source_true(self):
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig(
+                        gemini_agent="projects/p/locations/l/agents/a"
+                    ),
+                    interaction="projects/p/locations/l/interactions/i1",
+                )
+            )
+        ]
+        assert _evals_common._has_interactions_data_source(cases)
+
+    def test_has_interactions_data_source_false(self):
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                prompt=genai_types.Content(
+                    parts=[genai_types.Part(text="test")]
+                ),
+            )
+        ]
+        assert not _evals_common._has_interactions_data_source(cases)
+
+    def test_resolve_rejects_mixed_cases(self):
+        """Mixing interaction-based and prompt-based cases raises ValueError."""
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    interaction="projects/p/locations/l/interactions/i1",
+                )
+            ),
+            agentplatform_genai_types.EvalCase(
+                prompt=genai_types.Content(
+                    parts=[genai_types.Part(text="test")]
+                ),
+            ),
+        ]
+        with pytest.raises(ValueError, match="interactions_data_source"):
+            _evals_common._resolve_interactions_to_eval_cases(
+                self.mock_api_client, cases
+            )
+
+    def test_resolve_rejects_missing_interaction(self):
+        """EvalCase with interactions_data_source but no interaction raises."""
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig(
+                        gemini_agent="projects/p/locations/l/agents/a"
+                    ),
+                )
+            ),
+        ]
+        with pytest.raises(ValueError, match="interaction is required"):
+            _evals_common._resolve_interactions_to_eval_cases(
+                self.mock_api_client, cases
+            )
+
+    def test_interaction_dict_to_agent_data_text_conversation(self):
+        """Converts user_input + model_output steps to agent_data."""
+        interaction_dict = {
+            "steps": [
+                {
+                    "type": "user_input",
+                    "content": [{"type": "text", "text": "Hello agent"}],
+                },
+                {
+                    "type": "model_output",
+                    "content": [
+                        {"type": "text", "text": "Hello! How can I help?"}
+                    ],
+                },
+            ]
+        }
+
+        result = _evals_common._interaction_dict_to_agent_data(
+            interaction_dict
+        )
+
+        assert len(result["turns"]) == 1
+        events = result["turns"][0]["events"]
+        assert len(events) == 2
+        assert events[0]["author"] == "user"
+        assert events[0]["content"]["parts"][0]["text"] == "Hello agent"
+        assert events[1]["author"] == "agent"
+        assert events[1]["content"]["parts"][0]["text"] == (
+            "Hello! How can I help?"
+        )
+
+    def test_interaction_dict_to_agent_data_with_tool_calls(self):
+        """Converts function_call + function_result steps."""
+        interaction_dict = {
+            "steps": [
+                {
+                    "type": "function_call",
+                    "name": "get_weather",
+                    "arguments": {"city": "NYC"},
+                    "id": "call_1",
+                },
+                {
+                    "type": "function_result",
+                    "name": "get_weather",
+                    "callId": "call_1",
+                    "result": {"temp": "72F"},
+                },
+            ]
+        }
+
+        result = _evals_common._interaction_dict_to_agent_data(
+            interaction_dict
+        )
+
+        events = result["turns"][0]["events"]
+        assert len(events) == 2
+        fc_event = events[0]
+        assert fc_event["author"] == "agent"
+        assert fc_event["content"]["parts"][0]["function_call"]["name"] == (
+            "get_weather"
+        )
+        fr_event = events[1]
+        assert fr_event["content"]["parts"][0]["function_response"]["id"] == (
+            "call_1"
+        )
+
+
 class TestRateLimiter:
     """Tests for the RateLimiter class in _evals_utils."""