diff --git a/engine/hooks/llm-judge/README.md b/engine/hooks/llm-judge/README.md index 31eba814..ec80ca2d 100644 --- a/engine/hooks/llm-judge/README.md +++ b/engine/hooks/llm-judge/README.md @@ -165,7 +165,9 @@ hold up the reply. `inbox.messages(transcript)` drains that transcript's verdicts and turns each one into a line of text: -- **hit**: the job's `on_hit` text, word for word. +- **hit**: the job's `on_hit` text, word for word, followed by a space and the + answer's `report` string when the answer has a non-blank one, clipped to 600 + characters. - **unchecked**: `llm-judge: could not judge the last reply: ` then `: ` for each try, joined by `; `. If there were no tries (the judge broke, or the verdict file was unreadable), the verdict's own diff --git a/engine/hooks/llm-judge/inbox.py b/engine/hooks/llm-judge/inbox.py index fd9766fa..ed285123 100644 --- a/engine/hooks/llm-judge/inbox.py +++ b/engine/hooks/llm-judge/inbox.py @@ -6,6 +6,7 @@ import judge NO_TRANSCRIPT = "llm-judge: {harness} payload has no transcript path, so finished verdicts were not checked" +REPORT_LIMIT = 600 def resolve_transcript(payload: dict) -> str: @@ -49,6 +50,11 @@ def messages(transcript: str) -> list[str]: text = item.get("on_hit") if not isinstance(text, str) or not text.strip(): text = f"llm-judge: {item.get('hook') or 'unknown hook'} flagged the last reply: {item.get('reason')}" + answer = item.get("answer") + if isinstance(answer, dict): + report = answer.get("report") + if isinstance(report, str) and report.strip(): + text = f"{text} {report.strip()[:REPORT_LIMIT]}" out.append(text) continue out.append(unchecked_message(item)) diff --git a/engine/hooks/llm-judge/tests/test_inbox.py b/engine/hooks/llm-judge/tests/test_inbox.py index b142bdcb..af270bd4 100644 --- a/engine/hooks/llm-judge/tests/test_inbox.py +++ b/engine/hooks/llm-judge/tests/test_inbox.py @@ -56,6 +56,9 @@ def seed(self, *runner_entries, job_id="job-1"): }) return judge.run_job(job_path) + def seed_verdict(self, verdict, job_id="job-1"): + judge.write_json_atomic(os.path.join(judge.verdict_dir(self.transcript), f"{job_id}.json"), verdict) + def run_claude(self, stdin_text): out, err = io.StringIO(), io.StringIO() with patch.object(sys, "stdin", io.StringIO(stdin_text)), redirect_stdout(out), redirect_stderr(err): @@ -84,6 +87,31 @@ def test_hit_yields_the_exact_on_hit_text_once(self): self.assertEqual(inbox.messages(self.transcript), [ON_HIT]) self.assertEqual(inbox.messages(self.transcript), []) + def test_hit_with_report_appends_report(self): + report = "model saw a quoted rollback" + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}}) + found = inbox.messages(self.transcript) + self.assertEqual(found, [f"{ON_HIT} {report}"]) + self.assertTrue(found[0].endswith(f" {report}"), found) + + def test_hit_without_report_equals_on_hit_exactly(self): + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"match": True}}) + self.assertEqual(inbox.messages(self.transcript), [ON_HIT]) + + def test_hit_report_is_clipped_to_600_characters(self): + report = "x" * 700 + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}}) + self.assertEqual(inbox.messages(self.transcript), [f"{ON_HIT} {'x' * 600}"]) + + def test_hit_number_or_list_report_is_ignored(self): + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": 5}}, job_id="a") + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": ["detail"]}}, job_id="b") + self.assertEqual(inbox.messages(self.transcript), [ON_HIT, ON_HIT]) + + def test_clean_with_report_yields_nothing(self): + self.seed_verdict({"outcome": "clean", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": "ignored"}}) + self.assertEqual(inbox.messages(self.transcript), []) + def test_unchecked_yields_one_reason_per_runner(self): self.assertEqual(self.seed(MISSING, CRASHES)["outcome"], "unchecked") self.assertEqual(