merge queue: checking #540 on main (344657b), stacked on #519 and #525

corpus/skills/principle-prove-it/SKILL.md

@@ -34,6 +34,22 @@ check now, not lower the confidence and continue.
 **Absence of output is not proof of success.** A command that printed
 nothing needs its exit code shown.
 
+**The output must entail the sentence, not merely agree with it.** Before
+writing "verified," read the claim and the pasted output side by side and ask
+what the output actually rules out. A run on one version, one host, one image,
+one input proves the claim *for that instance*; it does not prove the general
+or version-boundary statement the sentence made. When the check comes back
+narrower than the claim — and it usually will, because the cheap check is the
+reachable instance — rewrite the claim down to what ran, and say the wider one
+is still open. Filing a narrow result under a wide heading is the error, even
+when every word of the output is true. This binds hardest in a correction:
+restating the original overclaim while pasting a narrower proof relabels the
+mistake as a fix. Named in logic as hasty generalization, *secundum quid*
+(Aristotle, *Sophistical Refutations*, Bk. I ch. 5, trans. W.A.
+Pickard-Cambridge, http://classics.mit.edu/Aristotle/sophist_refut.html); in
+software it is the difference between a witness and a proof, since one passing
+instance witnesses existence and never universality.
+
 **Blaming a gate is a causal claim.** "The hook is wrong," "the check
 misfired," "the classifier blocked it for no reason" — each one needs the
 gate's rule read this turn and quoted, with its `file:line`, next to the

corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md

@@ -0,0 +1,17 @@
+An agent runs `docker run --rm node:25-slim corepack --version` and pastes the
+real output: `corepack: not found`. It then writes "verified: corepack was
+removed from Node 25+" into the PR body. One image, one tag, one run. Nothing
+checked a second 25.x image, a later version, or the release notes.
+
+This skill fires. The pasted output is real and every word of it is true, but
+it does not entail the sentence. It rules out corepack in that one image; it
+says nothing about the version boundary the claim draws. The rule that the
+output must entail the sentence, not merely agree with it, is what the reply
+needs: rewrite the claim down to what ran ("the `node:25-slim` image has no
+corepack") and mark the wider statement as open, or run the check that would
+actually cover it.
+
+The same shape fires on a correction. If the agent later says "I overstated
+it earlier, here is the proof" and pastes the same single-image run under the
+same "removed from Node 25+" heading, the correction has relabeled the
+overclaim as a fix. The narrower proof needs the narrower sentence.

engine/hooks/llm-judge/README.md

@@ -165,7 +165,9 @@ hold up the reply.
 `inbox.messages(transcript)` drains that transcript's verdicts and turns each
 one into a line of text:
 
-- **hit**: the job's `on_hit` text, word for word.
+- **hit**: the job's `on_hit` text, word for word, followed by a space and the
+  answer's `report` string when the answer has a non-blank one, clipped to 600
+  characters.
 - **unchecked**: `llm-judge: <hook> could not judge the last reply: ` then
   `<runner>: <reason>` for each try, joined by `; `. If there were no tries
   (the judge broke, or the verdict file was unreadable), the verdict's own

engine/hooks/llm-judge/inbox.py

@@ -6,6 +6,7 @@
 import judge
 
 NO_TRANSCRIPT = "llm-judge: {harness} payload has no transcript path, so finished verdicts were not checked"
+REPORT_LIMIT = 600
 
 
 def resolve_transcript(payload: dict) -> str:
@@ -49,6 +50,11 @@ def messages(transcript: str) -> list[str]:
             text = item.get("on_hit")
             if not isinstance(text, str) or not text.strip():
                 text = f"llm-judge: {item.get('hook') or 'unknown hook'} flagged the last reply: {item.get('reason')}"
+            answer = item.get("answer")
+            if isinstance(answer, dict):
+                report = answer.get("report")
+                if isinstance(report, str) and report.strip():
+                    text = f"{text} {report.strip()[:REPORT_LIMIT]}"
             out.append(text)
             continue
         out.append(unchecked_message(item))

engine/hooks/llm-judge/judge_test_base.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import sys
 import tempfile
 import unittest
 from unittest.mock import patch
@@ -13,10 +14,14 @@ class JudgeTestCase(unittest.TestCase):
     def setUp(self):
         super().setUp()
         self.state = tempfile.TemporaryDirectory()
-        self.judge_env = patch.dict(os.environ, {judge.STATE_ENV: self.state.name})
+        self.judge_env = patch.dict(os.environ, {
+            judge.STATE_ENV: self.state.name,
+            judge.RUNNERS_ENV: json.dumps([
+                ["stub", [sys.executable, "-c", "print('{\"match\": false}')", judge.PROMPT_SLOT]],
+            ]),
+        })
         self.judge_env.start()
         os.environ.pop(judge.CHILD_ENV, None)
-        os.environ.pop(judge.RUNNERS_ENV, None)
 
     def tearDown(self):
         self.judge_env.stop()

engine/hooks/llm-judge/tests/test_inbox.py

@@ -56,6 +56,9 @@ def seed(self, *runner_entries, job_id="job-1"):
         })
         return judge.run_job(job_path)
 
+    def seed_verdict(self, verdict, job_id="job-1"):
+        judge.write_json_atomic(os.path.join(judge.verdict_dir(self.transcript), f"{job_id}.json"), verdict)
+
     def run_claude(self, stdin_text):
         out, err = io.StringIO(), io.StringIO()
         with patch.object(sys, "stdin", io.StringIO(stdin_text)), redirect_stdout(out), redirect_stderr(err):
@@ -84,6 +87,31 @@ def test_hit_yields_the_exact_on_hit_text_once(self):
         self.assertEqual(inbox.messages(self.transcript), [ON_HIT])
         self.assertEqual(inbox.messages(self.transcript), [])
 
+    def test_hit_with_report_appends_report(self):
+        report = "model saw a quoted rollback"
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}})
+        found = inbox.messages(self.transcript)
+        self.assertEqual(found, [f"{ON_HIT} {report}"])
+        self.assertTrue(found[0].endswith(f" {report}"), found)
+
+    def test_hit_without_report_equals_on_hit_exactly(self):
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"match": True}})
+        self.assertEqual(inbox.messages(self.transcript), [ON_HIT])
+
+    def test_hit_report_is_clipped_to_600_characters(self):
+        report = "x" * 700
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}})
+        self.assertEqual(inbox.messages(self.transcript), [f"{ON_HIT} {'x' * 600}"])
+
+    def test_hit_number_or_list_report_is_ignored(self):
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": 5}}, job_id="a")
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": ["detail"]}}, job_id="b")
+        self.assertEqual(inbox.messages(self.transcript), [ON_HIT, ON_HIT])
+
+    def test_clean_with_report_yields_nothing(self):
+        self.seed_verdict({"outcome": "clean", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": "ignored"}})
+        self.assertEqual(inbox.messages(self.transcript), [])
+
     def test_unchecked_yields_one_reason_per_runner(self):
         self.assertEqual(self.seed(MISSING, CRASHES)["outcome"], "unchecked")
         self.assertEqual(

engine/hooks/llm-judge/tests/test_judge.py

@@ -108,10 +108,17 @@ def test_malformed_runners_env_refuses_instead_of_running_defaults(self):
         with self.assertRaises(ValueError):
             judge.ask("x")
 
+    def test_test_base_runs_only_the_local_stub(self):
+        self.assertEqual([name for name, _ in judge.runners()], ["stub"])
+        self.assertEqual(judge.ask("x")["answer"], {"match": False})
+
     def test_default_runner_order_is_codex_then_claude_then_cursor(self):
-        self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"])
+        with patch.dict(os.environ):
+            os.environ.pop(judge.RUNNERS_ENV)
+            self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"])
 
     def test_investigate_runner_argv_is_read_only_and_excludes_cursor(self):
+        os.environ.pop(judge.RUNNERS_ENV)
         self.assertEqual(
             judge.runners("investigate"),
             [
@@ -159,6 +166,7 @@ def test_investigate_runners_env_replaces_investigate_defaults(self):
         self.assertEqual(judge.runners("investigate"), [("probe", custom[1])])
 
     def test_investigate_job_threads_timeout_and_cwd_to_runner(self):
+        os.environ.pop(judge.RUNNERS_ENV)
         path = os.path.join(self.state.name, "jobs", "investigate-job.json")
         with tempfile.TemporaryDirectory() as cwd:
             judge.write_json_atomic(path, self.job(id="investigate-job", mode="investigate", timeout_seconds=123, cwd=cwd))
@@ -220,6 +228,18 @@ def test_unchecked_when_ask_was_unchecked(self):
 
 
 class TestBackground(JudgeBehaviorTestCase):
+    def test_enqueue_writes_only_to_temporary_state_directory(self):
+        with tempfile.TemporaryDirectory() as home:
+            with patch.dict(os.environ, {"HOME": home}):
+                with patch.dict(os.environ):
+                    os.environ.pop(judge.STATE_ENV)
+                    default_state = judge.state_root()
+                os.makedirs(default_state)
+                with patch.object(judge.subprocess, "Popen"):
+                    self.assertEqual(judge.enqueue(self.job(id="isolated-job")), "isolated-job")
+            self.assertTrue(os.path.isfile(os.path.join(self.state.name, "jobs", "isolated-job.json")))
+            self.assertEqual(os.listdir(default_state), [])
+
     def test_enqueue_as_judge_child_returns_none_and_starts_nothing(self):
         os.environ[judge.CHILD_ENV] = "1"
         self.use_runners(ANSWER_MATCH)