diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index 608d5a1db7..759862fc63 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -400,7 +400,7 @@ def _is_n_plus_1_inference( def _extract_response_from_completed_trace( agent_data: types.evals.AgentData, ) -> list[dict[str, Any]]: - """Extracts all events from a completed agent trace as event dicts. + """Extracts all events from a completed agent trace as event dicts. For BYOD (bring-your-own-data) use cases where the agent trace is already complete, this returns all events formatted as a list of @@ -408,16 +408,179 @@ def _extract_response_from_completed_trace( last element is the final agent response; preceding elements become intermediate events. """ - event_dicts: list[dict[str, Any]] = [] - for turn in agent_data.turns or []: - if not turn.events: - continue - for event in turn.events: - d: dict[str, Any] = {"author": event.author or "agent"} - if event.content: - d[CONTENT] = event.content.model_dump(exclude_none=True) - event_dicts.append(d) - return event_dicts + event_dicts: list[dict[str, Any]] = [] + for turn in agent_data.turns or []: + if not turn.events: + continue + for event in turn.events: + d: dict[str, Any] = {"author": event.author or "agent"} + if event.content: + d[CONTENT] = event.content.model_dump(exclude_none=True) + event_dicts.append(d) + return event_dicts + + +def _has_interactions_data_source( + eval_cases: list[types.EvalCase], +) -> bool: + """Returns True if any EvalCase has interactions_data_source set.""" + return any( + getattr(case, "interactions_data_source", None) is not None + for case in eval_cases + ) + + +def _interaction_dict_to_agent_data( + interaction: dict[str, Any], +) -> dict[str, Any]: + """Converts an Interaction API JSON response to an AgentData-compatible dict. + + Maps the flat list of Interaction steps (user_input, model_output, + function_call, function_result, etc.) into a single ConversationTurn + with AgentEvents, matching the AgentData structure expected by the + evaluation pipeline. + + Args: + interaction: A dict from the Interactions API GET response. + + Returns: + A dict matching the AgentData schema with a single turn containing + all events from the interaction. + """ + events = [] + for step in interaction.get("steps", []): + step_type = step.get("type") + + if step_type == "user_input": + parts = [] + for content_item in step.get("content", []): + if content_item.get("type") == "text": + parts.append({"text": content_item.get("text", "")}) + if parts: + events.append({ + "author": "user", + "content": {"role": "user", "parts": parts}, + }) + + elif step_type == "model_output": + parts = [] + for content_item in step.get("content", []): + if content_item.get("type") == "text": + parts.append({"text": content_item.get("text", "")}) + if parts: + events.append({ + "author": "agent", + "content": {"role": "model", "parts": parts}, + }) + + elif step_type == "function_call": + events.append({ + "author": "agent", + "content": { + "role": "model", + "parts": [{ + "function_call": { + "name": step.get("name", ""), + "args": step.get("arguments", {}), + "id": step.get("id", ""), + }, + }], + }, + }) + + elif step_type == "function_result": + result = step.get("result") + if isinstance(result, dict): + result_str = json.dumps(result) + elif isinstance(result, str): + result_str = result + else: + result_str = str(result) if result is not None else "" + events.append({ + "author": "user", + "content": { + "role": "user", + "parts": [{ + "function_response": { + "name": step.get("name", ""), + "response": {"result": result_str}, + "id": step.get("callId", step.get("call_id", "")), + }, + }], + }, + }) + + agent_data: dict[str, Any] = { + "turns": [{ + "turn_index": 0, + "events": events, + }], + } + return agent_data + + +def _resolve_interactions_to_eval_cases( + api_client: BaseApiClient, + eval_cases: list[types.EvalCase], +) -> list[types.EvalCase]: + """Resolves EvalCases with interactions_data_source to agent_data. + + For each EvalCase that has interactions_data_source set, fetches the + Interaction via the SDK's interactions.get() API, converts the steps + to AgentData, and returns a new EvalCase with agent_data populated. + + Args: + api_client: The API client (must have an interactions module). + eval_cases: EvalCases with interactions_data_source set. + + Returns: + New list of EvalCases with agent_data populated from resolved + interactions. + + Raises: + ValueError: If eval_cases have missing interaction references. + """ + # Validate all cases up front before making any API calls. + for case in eval_cases: + ids = case.interactions_data_source + if ids is None: + raise ValueError( + "All eval_cases must have interactions_data_source set when" + " using interaction resolution. Found a case without it. Do" + " not mix interaction-based and prompt-based eval cases." + ) + if not ids.interaction: + raise ValueError( + "interactions_data_source.interaction is required. Each" + " EvalCase must reference an existing Interaction resource." + ) + + resolved_cases = [] + + for case in eval_cases: + ids = case.interactions_data_source + + # Extract the interaction ID from the resource name. + # Format: projects/{p}/locations/{l}/interactions/{id} + parts = ids.interaction.split("/") + if len(parts) >= 6 and parts[4] == "interactions": + interaction_id = parts[5] + else: + interaction_id = ids.interaction + + logger.info("Fetching interaction: %s", ids.interaction) + path = f"interactions/{interaction_id}" + response = api_client.request("get", path, {}, None) + interaction_dict = ( + {} if not response.body else json.loads(response.body) + ) + + agent_data = _interaction_dict_to_agent_data(interaction_dict) + resolved_cases.append( + types.EvalCase(agent_data=agent_data) + ) + + return resolved_cases def _resolve_dataset( @@ -426,22 +589,31 @@ def _resolve_dataset( dest: str, parsed_agent_info: Optional[types.evals.AgentInfo] = None, ) -> types.EvaluationRunDataSource: - """Resolves dataset for the evaluation run.""" - if isinstance(dataset, types.EvaluationDataset): - candidate_name = _get_candidate_name(dataset, parsed_agent_info) - eval_df = dataset.eval_dataset_df - if eval_df is None and dataset.eval_cases: - eval_df = _eval_cases_to_dataframe(dataset.eval_cases) - - eval_set = _create_evaluation_set_from_dataframe( - api_client, - dest, - eval_df, - candidate_name, - parsed_agent_info=parsed_agent_info, - ) - dataset = types.EvaluationRunDataSource(evaluation_set=eval_set.name) - return dataset + """Resolves dataset for the evaluation run.""" + if isinstance(dataset, types.EvaluationDataset): + # Resolve EvalCases with interactions_data_source by fetching + # each interaction and converting it to agent_data, then flowing + # through the normal DataFrame/GCS pipeline. + if dataset.eval_cases and _has_interactions_data_source(dataset.eval_cases): + resolved_cases = _resolve_interactions_to_eval_cases( + api_client, dataset.eval_cases + ) + dataset = types.EvaluationDataset(eval_cases=resolved_cases) + + candidate_name = _get_candidate_name(dataset, parsed_agent_info) + eval_df = dataset.eval_dataset_df + if eval_df is None and dataset.eval_cases: + eval_df = _eval_cases_to_dataframe(dataset.eval_cases) + + eval_set = _create_evaluation_set_from_dataframe( + api_client, + dest, + eval_df, + candidate_name, + parsed_agent_info=parsed_agent_info, + ) + dataset = types.EvaluationRunDataSource(evaluation_set=eval_set.name) + return dataset def _get_default_prompt_template( diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index eda21f7e31..708a7d1825 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -9563,6 +9563,139 @@ def test_resolve_dataset_preserves_conversation_history( assert "conversation_history" in ptd_values +class TestResolveDatasetWithInteractions: + """Tests for resolving interactions_data_source in _resolve_dataset.""" + + def setup_method(self): + self.mock_api_client = mock.Mock() + self.mock_api_client.project = "test-project" + self.mock_api_client.location = "us-central1" + + def test_has_interactions_data_source_true(self): + cases = [ + agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig( + gemini_agent="projects/p/locations/l/agents/a" + ), + interaction="projects/p/locations/l/interactions/i1", + ) + ) + ] + assert _evals_common._has_interactions_data_source(cases) + + def test_has_interactions_data_source_false(self): + cases = [ + agentplatform_genai_types.EvalCase( + prompt=genai_types.Content( + parts=[genai_types.Part(text="test")] + ), + ) + ] + assert not _evals_common._has_interactions_data_source(cases) + + def test_resolve_rejects_mixed_cases(self): + """Mixing interaction-based and prompt-based cases raises ValueError.""" + cases = [ + agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + interaction="projects/p/locations/l/interactions/i1", + ) + ), + agentplatform_genai_types.EvalCase( + prompt=genai_types.Content( + parts=[genai_types.Part(text="test")] + ), + ), + ] + with pytest.raises(ValueError, match="interactions_data_source"): + _evals_common._resolve_interactions_to_eval_cases( + self.mock_api_client, cases + ) + + def test_resolve_rejects_missing_interaction(self): + """EvalCase with interactions_data_source but no interaction raises.""" + cases = [ + agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig( + gemini_agent="projects/p/locations/l/agents/a" + ), + ) + ), + ] + with pytest.raises(ValueError, match="interaction is required"): + _evals_common._resolve_interactions_to_eval_cases( + self.mock_api_client, cases + ) + + def test_interaction_dict_to_agent_data_text_conversation(self): + """Converts user_input + model_output steps to agent_data.""" + interaction_dict = { + "steps": [ + { + "type": "user_input", + "content": [{"type": "text", "text": "Hello agent"}], + }, + { + "type": "model_output", + "content": [ + {"type": "text", "text": "Hello! How can I help?"} + ], + }, + ] + } + + result = _evals_common._interaction_dict_to_agent_data( + interaction_dict + ) + + assert len(result["turns"]) == 1 + events = result["turns"][0]["events"] + assert len(events) == 2 + assert events[0]["author"] == "user" + assert events[0]["content"]["parts"][0]["text"] == "Hello agent" + assert events[1]["author"] == "agent" + assert events[1]["content"]["parts"][0]["text"] == ( + "Hello! How can I help?" + ) + + def test_interaction_dict_to_agent_data_with_tool_calls(self): + """Converts function_call + function_result steps.""" + interaction_dict = { + "steps": [ + { + "type": "function_call", + "name": "get_weather", + "arguments": {"city": "NYC"}, + "id": "call_1", + }, + { + "type": "function_result", + "name": "get_weather", + "callId": "call_1", + "result": {"temp": "72F"}, + }, + ] + } + + result = _evals_common._interaction_dict_to_agent_data( + interaction_dict + ) + + events = result["turns"][0]["events"] + assert len(events) == 2 + fc_event = events[0] + assert fc_event["author"] == "agent" + assert fc_event["content"]["parts"][0]["function_call"]["name"] == ( + "get_weather" + ) + fr_event = events[1] + assert fr_event["content"]["parts"][0]["function_response"]["id"] == ( + "call_1" + ) + + class TestRateLimiter: """Tests for the RateLimiter class in _evals_utils."""