From db95a89ba179568f33e0c1cd9d188f5bdd460021 Mon Sep 17 00:00:00 2001 From: etserend Date: Mon, 17 Aug 2026 16:07:08 -0500 Subject: [PATCH 1/4] fix(converter): keep only last message from full-history agent output (HYBIM-988) --- src/splunk_ao/converter/attribute_mapping.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 64b782d3..90ec0561 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -447,6 +447,10 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: return if full_history and input_messages is not None and output_messages[: len(input_messages)] == input_messages: output_messages = output_messages[len(input_messages) :] + # Full history includes all messages in the run, not just the final response. + # Keep only the last message — it is always the agent's final output. + if full_history and len(output_messages) > 1: + output_messages = [output_messages[-1]] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) From 2ad296269a97fef9c8e94d74a1b61869b8399ad1 Mon Sep 17 00:00:00 2001 From: etserend Date: Mon, 17 Aug 2026 17:11:23 -0500 Subject: [PATCH 2/4] test(converter): add edge case tests for full-history agent output reduction (HYBIM-988) --- tests/test_attribute_mapping.py | 84 +++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 31571bfa..1b629b81 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -418,6 +418,90 @@ def test_orchestration_output_omits_repeated_input_history() -> None: ] +def test_orchestration_full_history_with_tool_call_keeps_last_message() -> None: + # LangGraph accumulated state: user → tool-call AI (empty content) → tool response → final AI + # The first post-dedup message has empty content; the UI would show "—" without the fix. + user = {"role": "user", "content": "What is the dosage of Lisinopril?"} + ai_toolcall = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "rag_search", "arguments": '{"query":"Lisinopril dosage"}'}}]} + tool_resp = {"role": "tool", "content": "Lisinopril: 10mg daily", "tool_call_id": "tc1"} + ai_final = {"role": "assistant", "content": "Common dosage is 10mg once daily."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, ai_toolcall, tool_resp, ai_final]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["parts"][0]["content"] == "Common dosage is 10mg once daily." + + +def test_orchestration_full_history_multi_turn_keeps_last_message() -> None: + # Multi-turn: output contains the full conversation history after multiple exchanges. + # Only the last message should be kept regardless of role. + user1 = {"role": "user", "content": "Hello"} + ai1 = {"role": "assistant", "content": "Hi, how can I help?"} + user2 = {"role": "user", "content": "What is Lisinopril?"} + ai2 = {"role": "assistant", "content": "Lisinopril is a blood pressure medication."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user1]}), + output=json.dumps({"messages": [user1, ai1, user2, ai2]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["parts"][0]["content"] == "Lisinopril is a blood pressure medication." + + +def test_orchestration_full_history_multiple_tool_rounds_keeps_last_message() -> None: + # Two tool call rounds before the final answer — last message is still the only output. + user = {"role": "user", "content": "Compare Lisinopril and Amlodipine"} + tc1_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}]} + tc1_resp = {"role": "tool", "content": "Lisinopril: ACE inhibitor", "tool_call_id": "tc1"} + tc2_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc2", "function": {"name": "search", "arguments": '{"query":"Amlodipine"}'}}]} + tc2_resp = {"role": "tool", "content": "Amlodipine: calcium channel blocker", "tool_call_id": "tc2"} + ai_final = {"role": "assistant", "content": "Lisinopril is an ACE inhibitor; Amlodipine is a calcium channel blocker."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, tc1_ai, tc1_resp, tc2_ai, tc2_resp, ai_final]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert "Amlodipine" in output_messages[0]["parts"][0]["content"] + + +def test_orchestration_non_full_history_output_not_reduced() -> None: + # Plain string output (full_history=False) — the last-message reduction must NOT fire. + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input="What is Lisinopril?", + output="Lisinopril is a blood pressure medication.", + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["parts"][0]["content"] == "Lisinopril is a blood pressure medication." + + def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: span = WorkflowSpan( name="tool-workflow", From b727b9180871f1772ec30477e7969e70ae95ce88 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 18 Aug 2026 10:22:30 -0500 Subject: [PATCH 3/4] fix(converter): preserve agent and workflow outputs --- CHANGELOG.md | 7 + src/splunk_ao/converter/attribute_mapping.py | 18 ++- tests/test_attribute_mapping.py | 144 ++++++++++++++++++- 3 files changed, 158 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b750ee4..de91d4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Agent and workflow output conversion now removes only confirmed repeated input + history, preserves multiple terminal assistant messages, and reports the + standard `tool_call` finish reason when an output requests a tool and no + source finish reason is available. + ## [0.2.1] - 2026-08-07 ### Fixed diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 90ec0561..f3929270 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -268,7 +268,12 @@ def _orchestration_messages(value: Any, default_role: str) -> tuple[list[dict[st def _with_finish_reasons(messages: list[dict[str, Any]], finish_reason: str | None = None) -> list[dict[str, Any]]: for message in messages: source_finish_reason = message.get("finish_reason") - message["finish_reason"] = finish_reason or source_finish_reason or "unknown" + inferred_finish_reason = ( + "tool_call" + if any(part.get("type") == "tool_call" for part in message.get("parts", []) if isinstance(part, Mapping)) + else "unknown" + ) + message["finish_reason"] = finish_reason or source_finish_reason or inferred_finish_reason return messages @@ -445,12 +450,13 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: output_messages, full_history = _orchestration_messages(span.output, "assistant") if output_messages is None: return - if full_history and input_messages is not None and output_messages[: len(input_messages)] == input_messages: + if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: output_messages = output_messages[len(input_messages) :] - # Full history includes all messages in the run, not just the final response. - # Keep only the last message — it is always the agent's final output. - if full_history and len(output_messages) > 1: - output_messages = [output_messages[-1]] + + terminal_start = len(output_messages) + while terminal_start > 0 and output_messages[terminal_start - 1].get("role") == "assistant": + terminal_start -= 1 + output_messages = output_messages[terminal_start:] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 1b629b81..5e0b01de 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -1,6 +1,6 @@ import json from types import SimpleNamespace -from typing import cast +from typing import Any, cast from uuid import uuid4 import pytest @@ -418,11 +418,134 @@ def test_orchestration_output_omits_repeated_input_history() -> None: ] +@pytest.mark.parametrize("span_type", [WorkflowSpan, AgentSpan]) +def test_orchestration_full_history_preserves_all_terminal_assistant_messages( + span_type: type[WorkflowSpan] | type[AgentSpan], +) -> None: + # Given: a full-history result with two terminal assistant outputs after the exact input history. + user = {"role": "user", "content": "Give me two alternatives"} + first = {"role": "assistant", "content": "First alternative"} + second = {"role": "assistant", "content": "Second alternative"} + span_kwargs: dict[str, Any] = { + "name": "planner", + "input": json.dumps({"messages": [user]}), + "output": json.dumps({"messages": [user, first, second]}), + } + if span_type is AgentSpan: + span_kwargs["agent_type"] = AgentType.planner + + # When: the orchestration content is converted. + attrs = build_span_attributes(span_type(**span_kwargs)) + + # Then: the repeated input prefix is removed without reducing the terminal outputs to one message. + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "First alternative", finish_reason="unknown"), + _text_message("assistant", "Second alternative", finish_reason="unknown"), + ] + + +@pytest.mark.parametrize("span_type", [WorkflowSpan, AgentSpan]) +def test_orchestration_full_history_removes_confirmed_input_prefix( + span_type: type[WorkflowSpan] | type[AgentSpan], +) -> None: + # Given: the input is the complete history immediately before the final assistant response. + user = {"role": "user", "content": "What is the dosage?"} + tool_call = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + tool_response = {"role": "tool", "content": "10 mg daily", "tool_call_id": "call-1"} + final = {"role": "assistant", "content": "The common dosage is 10 mg daily."} + input_history = [user, tool_call, tool_response] + span_kwargs: dict[str, Any] = { + "name": "healthcare", + "input": json.dumps({"messages": input_history}), + "output": json.dumps({"messages": [*input_history, final]}), + } + if span_type is AgentSpan: + span_kwargs["agent_type"] = AgentType.default + + # When: the orchestration content is converted. + attrs = build_span_attributes(span_type(**span_kwargs)) + + # Then: only the newly produced terminal response is exported as output. + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "The common dosage is 10 mg daily.", finish_reason="unknown") + ] + + +def test_orchestration_infers_tool_call_finish_reason_when_absent() -> None: + # Given: a workflow emits an assistant tool call without a source finish reason. + output = { + "update": { + "messages": [ + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + ] + } + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: the standard tool-call finish reason is inferred from the output part. + output_message = json.loads(attrs["gen_ai.output.messages"])[0] + assert output_message["finish_reason"] == "tool_call" + assert output_message["parts"][0]["type"] == "tool_call" + + +def test_orchestration_tool_response_uses_unknown_finish_reason() -> None: + # Given: a workflow emits a tool response, which has no model-generation finish reason. + output = { + "update": {"messages": [{"role": "tool", "content": {"dosage": "10 mg daily"}, "tool_call_id": "call-1"}]} + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: its valid tool response structure is preserved without inventing a model finish reason. + output_message = json.loads(attrs["gen_ai.output.messages"])[0] + assert output_message["finish_reason"] == "unknown" + assert output_message["parts"] == [ + {"type": "tool_call_response", "id": "call-1", "response": {"dosage": "10 mg daily"}} + ] + + +def test_orchestration_preserves_explicit_finish_reason_for_tool_call() -> None: + # Given: the source supplies its own finish reason for a message containing a tool call. + output = { + "update": { + "messages": [ + { + "role": "assistant", + "content": "", + "finish_reason": "provider_tool_calls", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + ] + } + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: inference does not overwrite source telemetry. + assert json.loads(attrs["gen_ai.output.messages"])[0]["finish_reason"] == "provider_tool_calls" + + def test_orchestration_full_history_with_tool_call_keeps_last_message() -> None: # LangGraph accumulated state: user → tool-call AI (empty content) → tool response → final AI # The first post-dedup message has empty content; the UI would show "—" without the fix. user = {"role": "user", "content": "What is the dosage of Lisinopril?"} - ai_toolcall = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "rag_search", "arguments": '{"query":"Lisinopril dosage"}'}}]} + ai_toolcall = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "rag_search", "arguments": '{"query":"Lisinopril dosage"}'}}], + } tool_resp = {"role": "tool", "content": "Lisinopril: 10mg daily", "tool_call_id": "tc1"} ai_final = {"role": "assistant", "content": "Common dosage is 10mg once daily."} @@ -466,11 +589,22 @@ def test_orchestration_full_history_multi_turn_keeps_last_message() -> None: def test_orchestration_full_history_multiple_tool_rounds_keeps_last_message() -> None: # Two tool call rounds before the final answer — last message is still the only output. user = {"role": "user", "content": "Compare Lisinopril and Amlodipine"} - tc1_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}]} + tc1_ai = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}], + } tc1_resp = {"role": "tool", "content": "Lisinopril: ACE inhibitor", "tool_call_id": "tc1"} - tc2_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc2", "function": {"name": "search", "arguments": '{"query":"Amlodipine"}'}}]} + tc2_ai = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc2", "function": {"name": "search", "arguments": '{"query":"Amlodipine"}'}}], + } tc2_resp = {"role": "tool", "content": "Amlodipine: calcium channel blocker", "tool_call_id": "tc2"} - ai_final = {"role": "assistant", "content": "Lisinopril is an ACE inhibitor; Amlodipine is a calcium channel blocker."} + ai_final = { + "role": "assistant", + "content": "Lisinopril is an ACE inhibitor; Amlodipine is a calcium channel blocker.", + } span = AgentSpan( name="Agent", From 766d70795004e16d55f05e1b6df65a5a9524a612 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 18 Aug 2026 10:46:00 -0500 Subject: [PATCH 4/4] test(converter): add parallel tool node output coverage --- tests/test_attribute_mapping.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 5e0b01de..a0864e90 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -620,20 +620,24 @@ def test_orchestration_full_history_multiple_tool_rounds_keeps_last_message() -> assert "Amlodipine" in output_messages[0]["parts"][0]["content"] -def test_orchestration_non_full_history_output_not_reduced() -> None: - # Plain string output (full_history=False) — the last-message reduction must NOT fire. - span = AgentSpan( - name="Agent", - agent_type=AgentType.default, - input="What is Lisinopril?", - output="Lisinopril is a blood pressure medication.", +def test_orchestration_message_container_without_input_prefix_match_not_reduced() -> None: + # A WorkflowSpan (e.g. ToolNode) returning multiple messages whose output does NOT + # prefix-match the input state — dedup gate never fires, so all messages must survive. + # This is the parallel-tool-call shape: two ToolMessages from a single ToolNode invocation. + tool_msg_1 = {"role": "tool", "content": "Lisinopril: 10 mg daily", "tool_call_id": "tc1"} + tool_msg_2 = {"role": "tool", "content": "Amlodipine: 5 mg daily", "tool_call_id": "tc2"} + span = WorkflowSpan( + name="tools", + input=json.dumps({"messages": [{"role": "user", "content": "Compare dosages"}]}), + output=json.dumps({"messages": [tool_msg_1, tool_msg_2]}), ) attrs = build_span_attributes(span) output_messages = json.loads(attrs["gen_ai.output.messages"]) - assert len(output_messages) == 1 - assert output_messages[0]["parts"][0]["content"] == "Lisinopril is a blood pressure medication." + assert len(output_messages) == 2 + assert output_messages[0]["parts"][0]["response"] == "Lisinopril: 10 mg daily" + assert output_messages[1]["parts"][0]["response"] == "Amlodipine: 5 mg daily" def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: