diff --git a/engine/hooks/agent-relay-attribution/detect.py b/engine/hooks/agent-relay-attribution/detect.py index 980a4aa4..731e617c 100644 --- a/engine/hooks/agent-relay-attribution/detect.py +++ b/engine/hooks/agent-relay-attribution/detect.py @@ -28,7 +28,7 @@ re.IGNORECASE, ) FACT_RE = re.compile(STATUS_RE.pattern + r"|\bbecause\b|\b\d+\b", re.IGNORECASE) -EVIDENCE_NUMBER_RE = re.compile(r"\b\d{2,}\b") +EVIDENCE_NUMBER_RE = re.compile(r"\b\d{3,}\b") ATTRIBUTION_RE = re.compile( r"\bper (?:the |its |their )?(?:sub)?agent'?s?(?: report| output| summary)?\b|" r"\bthe (?:sub)?agent (?:reported|reports|says|said|found|wrote|pasted|flagged)\b|" diff --git a/engine/hooks/agent-relay-attribution/tests/fixtures/relay_fires.json b/engine/hooks/agent-relay-attribution/tests/fixtures/relay_fires.json index 26e12fd3..2f1976be 100644 --- a/engine/hooks/agent-relay-attribution/tests/fixtures/relay_fires.json +++ b/engine/hooks/agent-relay-attribution/tests/fixtures/relay_fires.json @@ -165,5 +165,54 @@ } } ] + }, + { + "label": "two-digit tool output does not corroborate a relayed claim", + "reply": "The cheaper catch is our existing repeat-error-stop hook, which today resets its counter on every edit, so it stayed silent in both loops over the last 30 days.", + "transcript": [ + { + "type": "user", + "message": { + "role": "user", + "content": "plain text" + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": "\nrelay-30\ncompleted\nRelay completed\n" + } + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "t-relay-30", + "name": "Bash", + "input": { + "command": "python3 backtest.py --window 7" + } + } + ] + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "t-relay-30", + "type": "tool_result", + "content": "python3 backtest.py --window 7: timeout=30 ok" + } + ] + } + } + ] } ] diff --git a/engine/hooks/agent-relay-attribution/tests/fixtures/relay_silent.json b/engine/hooks/agent-relay-attribution/tests/fixtures/relay_silent.json index 5afaa3fb..ef11a198 100644 --- a/engine/hooks/agent-relay-attribution/tests/fixtures/relay_silent.json +++ b/engine/hooks/agent-relay-attribution/tests/fixtures/relay_silent.json @@ -1,4 +1,53 @@ [ + { + "label": "two-digit tool output remains unrelated to an attributed claim", + "reply": "UNVERIFIED: the subagent reports its counter resets on every edit, and it stayed silent over the last 30 days.", + "transcript": [ + { + "type": "user", + "message": { + "role": "user", + "content": "plain text" + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": "\nrelay-30\ncompleted\nRelay completed\n" + } + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "t-relay-30", + "name": "Bash", + "input": { + "command": "python3 backtest.py --window 7" + } + } + ] + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "t-relay-30", + "type": "tool_result", + "content": "python3 backtest.py --window 7: timeout=30 ok" + } + ] + } + } + ] + }, { "label": "same facts attributed with 'Per the agent's report' (real session)", "reply": "The gate-effectiveness suite is open as PR #12, not merged: https://github.com/owner/repo/pull/12\n\nPer the agent's report: 24 new tests, suite `271 passed in 14.78s`. One synthetic parent runs the real history, performance, export, reconcile, and grading code.", diff --git a/engine/hooks/agent-relay-attribution/tests/test_hooks.py b/engine/hooks/agent-relay-attribution/tests/test_hooks.py index 4bfaca18..d102f73d 100644 --- a/engine/hooks/agent-relay-attribution/tests/test_hooks.py +++ b/engine/hooks/agent-relay-attribution/tests/test_hooks.py @@ -60,6 +60,10 @@ def run_hook(payload): class TestFlagsUnattributedRelays(unittest.TestCase): + def test_two_digit_tool_output_does_not_corroborate_relay(self): + entry = case("relay_fires.json", "two-digit tool output does not corroborate") + self.assertIsNotNone(detect.decide_from_lines(entry["reply"], entry["transcript"])) + def test_flags_each_fires_fixture(self): for case in load("relay_fires.json"): with self.subTest(label=case["label"]): @@ -92,6 +96,10 @@ def test_flags_claim_absent_from_this_turns_command_output(self): class TestStaysSilentWhenAttributedOrVerified(unittest.TestCase): + def test_two_digit_tool_output_stays_unrelated_to_attributed_claim(self): + entry = case("relay_silent.json", "two-digit tool output remains unrelated") + self.assertIsNone(detect.decide_from_lines(entry["reply"], entry["transcript"])) + def test_silent_on_each_silent_fixture(self): for case in load("relay_silent.json"): with self.subTest(label=case["label"]):