diff --git a/corpus/skills/principle-prove-it/SKILL.md b/corpus/skills/principle-prove-it/SKILL.md index 237048ff..48fffdff 100644 --- a/corpus/skills/principle-prove-it/SKILL.md +++ b/corpus/skills/principle-prove-it/SKILL.md @@ -34,6 +34,22 @@ check now, not lower the confidence and continue. **Absence of output is not proof of success.** A command that printed nothing needs its exit code shown. +**The output must entail the sentence, not merely agree with it.** Before +writing "verified," read the claim and the pasted output side by side and ask +what the output actually rules out. A run on one version, one host, one image, +one input proves the claim *for that instance*; it does not prove the general +or version-boundary statement the sentence made. When the check comes back +narrower than the claim — and it usually will, because the cheap check is the +reachable instance — rewrite the claim down to what ran, and say the wider one +is still open. Filing a narrow result under a wide heading is the error, even +when every word of the output is true. This binds hardest in a correction: +restating the original overclaim while pasting a narrower proof relabels the +mistake as a fix. Named in logic as hasty generalization, *secundum quid* +(Aristotle, *Sophistical Refutations*, Bk. I ch. 5, trans. W.A. +Pickard-Cambridge, http://classics.mit.edu/Aristotle/sophist_refut.html); in +software it is the difference between a witness and a proof, since one passing +instance witnesses existence and never universality. + **Blaming a gate is a causal claim.** "The hook is wrong," "the check misfired," "the classifier blocked it for no reason" — each one needs the gate's rule read this turn and quoted, with its `file:line`, next to the diff --git a/corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md b/corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md new file mode 100644 index 00000000..844c4b67 --- /dev/null +++ b/corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md @@ -0,0 +1,17 @@ +An agent runs `docker run --rm node:25-slim corepack --version` and pastes the +real output: `corepack: not found`. It then writes "verified: corepack was +removed from Node 25+" into the PR body. One image, one tag, one run. Nothing +checked a second 25.x image, a later version, or the release notes. + +This skill fires. The pasted output is real and every word of it is true, but +it does not entail the sentence. It rules out corepack in that one image; it +says nothing about the version boundary the claim draws. The rule that the +output must entail the sentence, not merely agree with it, is what the reply +needs: rewrite the claim down to what ran ("the `node:25-slim` image has no +corepack") and mark the wider statement as open, or run the check that would +actually cover it. + +The same shape fires on a correction. If the agent later says "I overstated +it earlier, here is the proof" and pastes the same single-image run under the +same "removed from Node 25+" heading, the correction has relabeled the +overclaim as a fix. The narrower proof needs the narrower sentence. diff --git a/engine/hooks/llm-judge/README.md b/engine/hooks/llm-judge/README.md index 31eba814..ec80ca2d 100644 --- a/engine/hooks/llm-judge/README.md +++ b/engine/hooks/llm-judge/README.md @@ -165,7 +165,9 @@ hold up the reply. `inbox.messages(transcript)` drains that transcript's verdicts and turns each one into a line of text: -- **hit**: the job's `on_hit` text, word for word. +- **hit**: the job's `on_hit` text, word for word, followed by a space and the + answer's `report` string when the answer has a non-blank one, clipped to 600 + characters. - **unchecked**: `llm-judge: could not judge the last reply: ` then `: ` for each try, joined by `; `. If there were no tries (the judge broke, or the verdict file was unreadable), the verdict's own diff --git a/engine/hooks/llm-judge/inbox.py b/engine/hooks/llm-judge/inbox.py index fd9766fa..ed285123 100644 --- a/engine/hooks/llm-judge/inbox.py +++ b/engine/hooks/llm-judge/inbox.py @@ -6,6 +6,7 @@ import judge NO_TRANSCRIPT = "llm-judge: {harness} payload has no transcript path, so finished verdicts were not checked" +REPORT_LIMIT = 600 def resolve_transcript(payload: dict) -> str: @@ -49,6 +50,11 @@ def messages(transcript: str) -> list[str]: text = item.get("on_hit") if not isinstance(text, str) or not text.strip(): text = f"llm-judge: {item.get('hook') or 'unknown hook'} flagged the last reply: {item.get('reason')}" + answer = item.get("answer") + if isinstance(answer, dict): + report = answer.get("report") + if isinstance(report, str) and report.strip(): + text = f"{text} {report.strip()[:REPORT_LIMIT]}" out.append(text) continue out.append(unchecked_message(item)) diff --git a/engine/hooks/llm-judge/judge_test_base.py b/engine/hooks/llm-judge/judge_test_base.py index 01249d18..9bb41eb8 100644 --- a/engine/hooks/llm-judge/judge_test_base.py +++ b/engine/hooks/llm-judge/judge_test_base.py @@ -2,6 +2,7 @@ import json import os +import sys import tempfile import unittest from unittest.mock import patch @@ -13,10 +14,14 @@ class JudgeTestCase(unittest.TestCase): def setUp(self): super().setUp() self.state = tempfile.TemporaryDirectory() - self.judge_env = patch.dict(os.environ, {judge.STATE_ENV: self.state.name}) + self.judge_env = patch.dict(os.environ, { + judge.STATE_ENV: self.state.name, + judge.RUNNERS_ENV: json.dumps([ + ["stub", [sys.executable, "-c", "print('{\"match\": false}')", judge.PROMPT_SLOT]], + ]), + }) self.judge_env.start() os.environ.pop(judge.CHILD_ENV, None) - os.environ.pop(judge.RUNNERS_ENV, None) def tearDown(self): self.judge_env.stop() diff --git a/engine/hooks/llm-judge/tests/test_inbox.py b/engine/hooks/llm-judge/tests/test_inbox.py index b142bdcb..af270bd4 100644 --- a/engine/hooks/llm-judge/tests/test_inbox.py +++ b/engine/hooks/llm-judge/tests/test_inbox.py @@ -56,6 +56,9 @@ def seed(self, *runner_entries, job_id="job-1"): }) return judge.run_job(job_path) + def seed_verdict(self, verdict, job_id="job-1"): + judge.write_json_atomic(os.path.join(judge.verdict_dir(self.transcript), f"{job_id}.json"), verdict) + def run_claude(self, stdin_text): out, err = io.StringIO(), io.StringIO() with patch.object(sys, "stdin", io.StringIO(stdin_text)), redirect_stdout(out), redirect_stderr(err): @@ -84,6 +87,31 @@ def test_hit_yields_the_exact_on_hit_text_once(self): self.assertEqual(inbox.messages(self.transcript), [ON_HIT]) self.assertEqual(inbox.messages(self.transcript), []) + def test_hit_with_report_appends_report(self): + report = "model saw a quoted rollback" + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}}) + found = inbox.messages(self.transcript) + self.assertEqual(found, [f"{ON_HIT} {report}"]) + self.assertTrue(found[0].endswith(f" {report}"), found) + + def test_hit_without_report_equals_on_hit_exactly(self): + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"match": True}}) + self.assertEqual(inbox.messages(self.transcript), [ON_HIT]) + + def test_hit_report_is_clipped_to_600_characters(self): + report = "x" * 700 + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}}) + self.assertEqual(inbox.messages(self.transcript), [f"{ON_HIT} {'x' * 600}"]) + + def test_hit_number_or_list_report_is_ignored(self): + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": 5}}, job_id="a") + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": ["detail"]}}, job_id="b") + self.assertEqual(inbox.messages(self.transcript), [ON_HIT, ON_HIT]) + + def test_clean_with_report_yields_nothing(self): + self.seed_verdict({"outcome": "clean", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": "ignored"}}) + self.assertEqual(inbox.messages(self.transcript), []) + def test_unchecked_yields_one_reason_per_runner(self): self.assertEqual(self.seed(MISSING, CRASHES)["outcome"], "unchecked") self.assertEqual( diff --git a/engine/hooks/llm-judge/tests/test_judge.py b/engine/hooks/llm-judge/tests/test_judge.py index 52246b69..a41cf24d 100644 --- a/engine/hooks/llm-judge/tests/test_judge.py +++ b/engine/hooks/llm-judge/tests/test_judge.py @@ -108,10 +108,17 @@ def test_malformed_runners_env_refuses_instead_of_running_defaults(self): with self.assertRaises(ValueError): judge.ask("x") + def test_test_base_runs_only_the_local_stub(self): + self.assertEqual([name for name, _ in judge.runners()], ["stub"]) + self.assertEqual(judge.ask("x")["answer"], {"match": False}) + def test_default_runner_order_is_codex_then_claude_then_cursor(self): - self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) + with patch.dict(os.environ): + os.environ.pop(judge.RUNNERS_ENV) + self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) def test_investigate_runner_argv_is_read_only_and_excludes_cursor(self): + os.environ.pop(judge.RUNNERS_ENV) self.assertEqual( judge.runners("investigate"), [ @@ -159,6 +166,7 @@ def test_investigate_runners_env_replaces_investigate_defaults(self): self.assertEqual(judge.runners("investigate"), [("probe", custom[1])]) def test_investigate_job_threads_timeout_and_cwd_to_runner(self): + os.environ.pop(judge.RUNNERS_ENV) path = os.path.join(self.state.name, "jobs", "investigate-job.json") with tempfile.TemporaryDirectory() as cwd: judge.write_json_atomic(path, self.job(id="investigate-job", mode="investigate", timeout_seconds=123, cwd=cwd)) @@ -220,6 +228,18 @@ def test_unchecked_when_ask_was_unchecked(self): class TestBackground(JudgeBehaviorTestCase): + def test_enqueue_writes_only_to_temporary_state_directory(self): + with tempfile.TemporaryDirectory() as home: + with patch.dict(os.environ, {"HOME": home}): + with patch.dict(os.environ): + os.environ.pop(judge.STATE_ENV) + default_state = judge.state_root() + os.makedirs(default_state) + with patch.object(judge.subprocess, "Popen"): + self.assertEqual(judge.enqueue(self.job(id="isolated-job")), "isolated-job") + self.assertTrue(os.path.isfile(os.path.join(self.state.name, "jobs", "isolated-job.json"))) + self.assertEqual(os.listdir(default_state), []) + def test_enqueue_as_judge_child_returns_none_and_starts_nothing(self): os.environ[judge.CHILD_ENV] = "1" self.use_runners(ANSWER_MATCH)