merge queue: checking #494 on main (344657b), stacked on 4 pull requests ahead

corpus/skills/principle-prove-it/SKILL.md

@@ -34,6 +34,22 @@ check now, not lower the confidence and continue.
 **Absence of output is not proof of success.** A command that printed
 nothing needs its exit code shown.
 
+**The output must entail the sentence, not merely agree with it.** Before
+writing "verified," read the claim and the pasted output side by side and ask
+what the output actually rules out. A run on one version, one host, one image,
+one input proves the claim *for that instance*; it does not prove the general
+or version-boundary statement the sentence made. When the check comes back
+narrower than the claim — and it usually will, because the cheap check is the
+reachable instance — rewrite the claim down to what ran, and say the wider one
+is still open. Filing a narrow result under a wide heading is the error, even
+when every word of the output is true. This binds hardest in a correction:
+restating the original overclaim while pasting a narrower proof relabels the
+mistake as a fix. Named in logic as hasty generalization, *secundum quid*
+(Aristotle, *Sophistical Refutations*, Bk. I ch. 5, trans. W.A.
+Pickard-Cambridge, http://classics.mit.edu/Aristotle/sophist_refut.html); in
+software it is the difference between a witness and a proof, since one passing
+instance witnesses existence and never universality.
+
 **Blaming a gate is a causal claim.** "The hook is wrong," "the check
 misfired," "the classifier blocked it for no reason" — each one needs the
 gate's rule read this turn and quoted, with its `file:line`, next to the

corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md

@@ -0,0 +1,17 @@
+An agent runs `docker run --rm node:25-slim corepack --version` and pastes the
+real output: `corepack: not found`. It then writes "verified: corepack was
+removed from Node 25+" into the PR body. One image, one tag, one run. Nothing
+checked a second 25.x image, a later version, or the release notes.
+
+This skill fires. The pasted output is real and every word of it is true, but
+it does not entail the sentence. It rules out corepack in that one image; it
+says nothing about the version boundary the claim draws. The rule that the
+output must entail the sentence, not merely agree with it, is what the reply
+needs: rewrite the claim down to what ran ("the `node:25-slim` image has no
+corepack") and mark the wider statement as open, or run the check that would
+actually cover it.
+
+The same shape fires on a correction. If the agent later says "I overstated
+it earlier, here is the proof" and pastes the same single-image run under the
+same "removed from Node 25+" heading, the correction has relabeled the
+overclaim as a fix. The narrower proof needs the narrower sentence.

engine/hooks/gh-write-verification/detect.py

@@ -63,6 +63,7 @@
 import json
 import os
 import re
+from typing import Optional, Tuple
 
 TRUST_PR_EDIT_ENV = "GH_WRITE_VERIFICATION_TRUST_PR_EDIT"
 
@@ -213,6 +214,7 @@ def self_match_message(hits: list[str]) -> str:
     r"|\bgit\s+branch\s+-r\s+--contains\b"
     r"|\bgit\s+branch\s+--contains\b[^\n]*\s-r\b"
 )
+LANDING_OK_RE = re.compile(r"(?m)^\s*OK:")
 VERIFY_SCRIPT_RELPATH = "gh-write-verification/verify_pr_landed_on_trunk.sh"
 
 UNVERIFIED_MERGE_MESSAGE = (
@@ -288,22 +290,33 @@ def silenced_mutations(raw_text: str) -> list[str]:
     return hits
 
 
-def _proves_landing(command: str, number: str | None) -> bool:
+CommandRecord = Tuple[str, Optional[str]]
+
+
+def _proves_landing(command: str, number: str | None, result: str | None) -> bool:
     """True when this command checks where a merge commit actually landed.
 
     An invocation of the shipped verification script must name the PR it is
-    vouching for; a hand-rolled ancestry check is accepted as written, since
-    it takes a commit sha rather than a PR number.
+    vouching for, and the paired tool result must report the passing verdict.
     """
     command = command or ""
     if not LANDING_PROOF_RE.search(command):
         return False
     if "verify_pr_landed_on_trunk" in command and number is not None:
-        return number in command
-    return True
+        if number not in command:
+            return False
+    return bool(result and LANDING_OK_RE.search(result))
+
+
+def _command_text(record: str | CommandRecord) -> str:
+    return record[0] if isinstance(record, tuple) else record
+
+
+def _command_result(record: str | CommandRecord) -> str | None:
+    return record[1] if isinstance(record, tuple) else None
 
 
-def merges_missing_landing_proof(commands: list[str]) -> list[str]:
+def merges_missing_landing_proof(commands: list[str | CommandRecord]) -> list[str]:
     """PR subjects merged in this turn with no landing check run afterwards.
 
     Returns the merged subjects (a PR number, or "the current branch's PR"
@@ -312,13 +325,17 @@ def merges_missing_landing_proof(commands: list[str]) -> list[str]:
     proof ran after the merge.
     """
     subjects: list[str] = []
-    for index, command in enumerate(commands):
+    for index, record in enumerate(commands):
+        command = _command_text(record)
         match = GH_PR_MERGE_RE.search(command or "")
         if not match:
             continue
         number = match.group("number")
         subject = f"PR #{number}" if number else "the current branch's PR"
-        if any(_proves_landing(later, number) for later in commands[index + 1:]):
+        if any(
+            _proves_landing(_command_text(later), number, _command_result(later))
+            for later in commands[index + 1:]
+        ):
             continue
         if subject not in subjects:
             subjects.append(subject)
@@ -359,14 +376,25 @@ def _text_content(data: dict) -> str:
     return ""
 
 
+def _tool_result_text(block: dict) -> str:
+    content = block.get("content")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(
+            b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text"
+        )
+    return ""
+
+
 def _is_human_user_line(data: dict) -> bool:
     if data.get("type") != "user":
         return False
     text = _text_content(data)
     return bool(text.strip()) and not text.lstrip().startswith("<")
 
 
-def bash_commands_this_turn(raw_lines) -> list[str]:
+def bash_commands_this_turn(raw_lines) -> list[CommandRecord]:
     """Bash tool commands issued since the last authored user message."""
     parsed: list[dict] = []
     for raw in raw_lines:
@@ -380,8 +408,21 @@ def bash_commands_this_turn(raw_lines) -> list[str]:
     for index, data in enumerate(parsed):
         if _is_human_user_line(data):
             turn_start = index
-    commands: list[str] = []
-    for data in parsed[turn_start:]:
+    records = parsed[turn_start:]
+    results: dict[str, str] = {}
+    for data in records:
+        message = data.get("message")
+        content = message.get("content") if isinstance(message, dict) else None
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict) or block.get("type") != "tool_result":
+                continue
+            tool_id = block.get("tool_use_id")
+            if tool_id:
+                results[str(tool_id)] = _tool_result_text(block)
+    commands: list[CommandRecord] = []
+    for data in records:
         if data.get("type") != "assistant":
             continue
         message = data.get("message")
@@ -395,7 +436,9 @@ def bash_commands_this_turn(raw_lines) -> list[str]:
                 continue
             tool_input = block.get("input")
             if isinstance(tool_input, dict):
-                commands.append(str(tool_input.get("command") or ""))
+                tool_id = block.get("id") or block.get("tool_use_id")
+                result = results.get(str(tool_id)) if tool_id else None
+                commands.append((str(tool_input.get("command") or ""), result))
     return commands
 
 

engine/hooks/gh-write-verification/tests/test_hooks.py

@@ -57,15 +57,24 @@ def bash_payload(command: str) -> dict:
     return {"tool_name": "Bash", "cwd": HOOK_DIR, "tool_input": {"command": command}}
 
 
-def transcript(commands: list[str]) -> str:
+def transcript(commands: list[str | tuple[str, str | None]]) -> str:
     lines = [json.dumps({"type": "user", "message": {"role": "user", "content": "land the stack"}})]
-    for command in commands:
+    for index, item in enumerate(commands):
+        command, result = item if isinstance(item, tuple) else (item, None)
+        tool_id = f"bash-{index}"
         lines.append(json.dumps({
             "type": "assistant",
             "message": {"content": [
-                {"type": "tool_use", "name": "Bash", "input": {"command": command}}
+                {"type": "tool_use", "id": tool_id, "name": "Bash", "input": {"command": command}}
             ]},
         }))
+        if result is not None:
+            lines.append(json.dumps({
+                "type": "user",
+                "message": {"content": [
+                    {"type": "tool_result", "tool_use_id": tool_id, "content": result}
+                ]},
+            }))
     handle = tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False)
     handle.write("\n".join(lines) + "\n")
     handle.close()
@@ -228,6 +237,9 @@ def test_entrypoint_denies_the_incident_wait_loop(self):
 
 
 class TestUnverifiedLanding(unittest.TestCase):
+    def proven(self, command: str) -> tuple[str, str]:
+        return (command, "OK: abc123 is an ancestor of origin/main\n")
+
     def test_a_merge_with_no_landing_check_is_flagged(self):
         self.assertEqual(
             merges_missing_landing_proof(["gh pr merge 291 --squash --admin", "gh pr view 291"]),
@@ -238,7 +250,7 @@ def test_every_unproven_merge_in_the_turn_is_flagged(self):
         commands = [
             "gh pr merge 291 --squash",
             "gh pr merge 292 --squash",
-            "bash verify_pr_landed_on_trunk.sh 292",
+            self.proven("bash verify_pr_landed_on_trunk.sh 292"),
         ]
         self.assertEqual(merges_missing_landing_proof(commands), ["PR #291"])
 
@@ -251,9 +263,18 @@ def test_a_verified_landing_stays_silent(self):
             "git branch -r --contains 314f0447",
         ):
             self.assertEqual(
-                merges_missing_landing_proof(["gh pr merge 291 --squash", proof]), [], proof
+                merges_missing_landing_proof(["gh pr merge 291 --squash", self.proven(proof)]), [], proof
             )
 
+    def test_a_landing_check_without_a_result_is_flagged(self):
+        self.assertEqual(
+            merges_missing_landing_proof([
+                "gh pr merge 291 --squash",
+                "bash verify_pr_landed_on_trunk.sh 291",
+            ]),
+            ["PR #291"],
+        )
+
     def test_a_turn_with_no_merge_stays_silent(self):
         self.assertEqual(merges_missing_landing_proof(["git status", "gh pr view 291"]), [])
 
@@ -271,14 +292,49 @@ def test_stop_entrypoint_denies_an_unproven_merge(self):
     def test_stop_entrypoint_allows_a_proven_merge(self):
         path = transcript([
             "gh pr merge 291 --squash --admin",
-            'bash "$HOME/.claude/hooks/gh-write-verification/verify_pr_landed_on_trunk.sh" 291',
+            (
+                'bash "$HOME/.claude/hooks/gh-write-verification/verify_pr_landed_on_trunk.sh" 291',
+                "pr=#291 repo=acme/widgets merged=true base=main merge_commit=abc123\n"
+                "OK: abc123 is an ancestor of origin/main\n",
+            ),
         ])
         try:
             result = run_entrypoint(STOP_CHECK, {"transcript_path": path})
             self.assertEqual(result.returncode, 0, result.stderr)
         finally:
             os.unlink(path)
 
+    def test_stop_entrypoint_denies_a_failed_landing_check(self):
+        path = transcript([
+            "gh pr merge 291 --squash --admin",
+            (
+                'bash "$HOME/.claude/hooks/gh-write-verification/verify_pr_landed_on_trunk.sh" 291',
+                "pr=#291 repo=acme/widgets merged=true base=stack merge_commit=abc123\n"
+                "FAIL: PR #291 in acme/widgets reports MERGED but abc123 is not on origin/main\n",
+            ),
+        ])
+        try:
+            result = run_entrypoint(STOP_CHECK, {"transcript_path": path})
+            self.assertEqual(result.returncode, 2, result.stderr)
+            self.assertIn("PR #291", result.stderr)
+        finally:
+            os.unlink(path)
+
+    def test_stop_entrypoint_denies_an_unchecked_landing_check(self):
+        path = transcript([
+            "gh pr merge 291 --squash --admin",
+            (
+                'bash "$HOME/.claude/hooks/gh-write-verification/verify_pr_landed_on_trunk.sh" 291',
+                "UNCHECKED: gh cannot resolve a repository here\n",
+            ),
+        ])
+        try:
+            result = run_entrypoint(STOP_CHECK, {"transcript_path": path})
+            self.assertEqual(result.returncode, 2, result.stderr)
+            self.assertIn("PR #291", result.stderr)
+        finally:
+            os.unlink(path)
+
     def test_a_missing_transcript_fails_open(self):
         self.assertIsNone(decide_stop({"transcript_path": "/nonexistent/transcript.jsonl"}))
         self.assertIsNone(decide_stop({}))

engine/hooks/llm-judge/README.md

@@ -165,7 +165,9 @@ hold up the reply.
 `inbox.messages(transcript)` drains that transcript's verdicts and turns each
 one into a line of text:
 
-- **hit**: the job's `on_hit` text, word for word.
+- **hit**: the job's `on_hit` text, word for word, followed by a space and the
+  answer's `report` string when the answer has a non-blank one, clipped to 600
+  characters.
 - **unchecked**: `llm-judge: <hook> could not judge the last reply: ` then
   `<runner>: <reason>` for each try, joined by `; `. If there were no tries
   (the judge broke, or the verdict file was unreadable), the verdict's own

engine/hooks/llm-judge/inbox.py

@@ -6,6 +6,7 @@
 import judge
 
 NO_TRANSCRIPT = "llm-judge: {harness} payload has no transcript path, so finished verdicts were not checked"
+REPORT_LIMIT = 600
 
 
 def resolve_transcript(payload: dict) -> str:
@@ -49,6 +50,11 @@ def messages(transcript: str) -> list[str]:
             text = item.get("on_hit")
             if not isinstance(text, str) or not text.strip():
                 text = f"llm-judge: {item.get('hook') or 'unknown hook'} flagged the last reply: {item.get('reason')}"
+            answer = item.get("answer")
+            if isinstance(answer, dict):
+                report = answer.get("report")
+                if isinstance(report, str) and report.strip():
+                    text = f"{text} {report.strip()[:REPORT_LIMIT]}"
             out.append(text)
             continue
         out.append(unchecked_message(item))

engine/hooks/llm-judge/judge_test_base.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import sys
 import tempfile
 import unittest
 from unittest.mock import patch
@@ -13,10 +14,14 @@ class JudgeTestCase(unittest.TestCase):
     def setUp(self):
         super().setUp()
         self.state = tempfile.TemporaryDirectory()
-        self.judge_env = patch.dict(os.environ, {judge.STATE_ENV: self.state.name})
+        self.judge_env = patch.dict(os.environ, {
+            judge.STATE_ENV: self.state.name,
+            judge.RUNNERS_ENV: json.dumps([
+                ["stub", [sys.executable, "-c", "print('{\"match\": false}')", judge.PROMPT_SLOT]],
+            ]),
+        })
         self.judge_env.start()
         os.environ.pop(judge.CHILD_ENV, None)
-        os.environ.pop(judge.RUNNERS_ENV, None)
 
     def tearDown(self):
         self.judge_env.stop()

engine/hooks/llm-judge/tests/test_inbox.py

@@ -56,6 +56,9 @@ def seed(self, *runner_entries, job_id="job-1"):
         })
         return judge.run_job(job_path)
 
+    def seed_verdict(self, verdict, job_id="job-1"):
+        judge.write_json_atomic(os.path.join(judge.verdict_dir(self.transcript), f"{job_id}.json"), verdict)
+
     def run_claude(self, stdin_text):
         out, err = io.StringIO(), io.StringIO()
         with patch.object(sys, "stdin", io.StringIO(stdin_text)), redirect_stdout(out), redirect_stderr(err):
@@ -84,6 +87,31 @@ def test_hit_yields_the_exact_on_hit_text_once(self):
         self.assertEqual(inbox.messages(self.transcript), [ON_HIT])
         self.assertEqual(inbox.messages(self.transcript), [])
 
+    def test_hit_with_report_appends_report(self):
+        report = "model saw a quoted rollback"
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}})
+        found = inbox.messages(self.transcript)
+        self.assertEqual(found, [f"{ON_HIT} {report}"])
+        self.assertTrue(found[0].endswith(f" {report}"), found)
+
+    def test_hit_without_report_equals_on_hit_exactly(self):
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"match": True}})
+        self.assertEqual(inbox.messages(self.transcript), [ON_HIT])
+
+    def test_hit_report_is_clipped_to_600_characters(self):
+        report = "x" * 700
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}})
+        self.assertEqual(inbox.messages(self.transcript), [f"{ON_HIT} {'x' * 600}"])
+
+    def test_hit_number_or_list_report_is_ignored(self):
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": 5}}, job_id="a")
+        self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": ["detail"]}}, job_id="b")
+        self.assertEqual(inbox.messages(self.transcript), [ON_HIT, ON_HIT])
+
+    def test_clean_with_report_yields_nothing(self):
+        self.seed_verdict({"outcome": "clean", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": "ignored"}})
+        self.assertEqual(inbox.messages(self.transcript), [])
+
     def test_unchecked_yields_one_reason_per_runner(self):
         self.assertEqual(self.seed(MISSING, CRASHES)["outcome"], "unchecked")
         self.assertEqual(