From eaa95047932bf611b31df321db0069632060bb44 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sat, 12 Sep 2026 20:01:53 -0700 Subject: [PATCH 01/13] [Hooks Cannot See] (2) Require the output to entail the claim, not merely agree The session behind slice (1) ran one command against one container image and proved that image had no corepack. That was written up as corepack having been removed from Node 25+, a version-boundary claim no command in the session ever checked. The output was true and the sentence was wider than the output. prove-it already covers a missing check and a hedge. It did not cover a check that ran, came back narrower than the claim, and got filed under the wider heading anyway -- which is the common case, because the cheap check is always the reachable instance. It binds hardest in a correction: restating the original overclaim while pasting a narrower proof relabels the mistake as a fix, which is what happened here before the rule existed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011ZqCxSQVpmM9Rx37C7irgf Change-Id: Ibc32514be641d37277115faf6068e75ea02e0083 --- corpus/skills/principle-prove-it/SKILL.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/corpus/skills/principle-prove-it/SKILL.md b/corpus/skills/principle-prove-it/SKILL.md index 237048ff..48fffdff 100644 --- a/corpus/skills/principle-prove-it/SKILL.md +++ b/corpus/skills/principle-prove-it/SKILL.md @@ -34,6 +34,22 @@ check now, not lower the confidence and continue. **Absence of output is not proof of success.** A command that printed nothing needs its exit code shown. +**The output must entail the sentence, not merely agree with it.** Before +writing "verified," read the claim and the pasted output side by side and ask +what the output actually rules out. A run on one version, one host, one image, +one input proves the claim *for that instance*; it does not prove the general +or version-boundary statement the sentence made. When the check comes back +narrower than the claim — and it usually will, because the cheap check is the +reachable instance — rewrite the claim down to what ran, and say the wider one +is still open. Filing a narrow result under a wide heading is the error, even +when every word of the output is true. This binds hardest in a correction: +restating the original overclaim while pasting a narrower proof relabels the +mistake as a fix. Named in logic as hasty generalization, *secundum quid* +(Aristotle, *Sophistical Refutations*, Bk. I ch. 5, trans. W.A. +Pickard-Cambridge, http://classics.mit.edu/Aristotle/sophist_refut.html); in +software it is the difference between a witness and a proof, since one passing +instance witnesses existence and never universality. + **Blaming a gate is a causal claim.** "The hook is wrong," "the check misfired," "the classifier blocked it for no reason" — each one needs the gate's rule read this turn and quoted, with its `file:line`, next to the From a6c7201ff31f9c7e80a83e982829cab3dc4e3485 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:04:58 +0000 Subject: [PATCH 02/13] =?UTF-8?q?invoker:=20wf-1789279240280-10/verify-jud?= =?UTF-8?q?ge-investigate-tests=20=E2=80=94=20Proof=20step=201=20for=20the?= =?UTF-8?q?=20llm-judge=20investigate=20mode.=20Review=20claim:=20The=20in?= =?UTF-8?q?vestigate-mode=20unit=20tests=20run=20and=20pass.=20Review=20la?= =?UTF-8?q?ne:=20proof=20Safety=20invariant:=20Proof-only;=20adds=20no=20p?= =?UTF-8?q?roduct=20behavior.=20Effectiveness=20measurement:=20The=20run?= =?UTF-8?q?=20exits=200=20and=20reports=20at=20least=20one=20test;=20Pytho?= =?UTF-8?q?n=20exits=205=20when=20-k=20matches=20nothing.=20Slice=20ration?= =?UTF-8?q?ale:=20One=20proof=20step=20for=20this=20workflow's=20single=20?= =?UTF-8?q?change.=20Architectural=20effect:=20None;=20verification=20only?= =?UTF-8?q?.=20Goal:=20Prove=20the=20investigate=20runner=20argv=20and=20t?= =?UTF-8?q?ime=20limit=20deterministically.=20Motivation:=20Each=20slice?= =?UTF-8?q?=20carries=20its=20own=20executable=20proof.=20Alternative=20co?= =?UTF-8?q?nsiderations:=20Manual=20inspection=20was=20rejected=20as=20non?= =?UTF-8?q?-deterministic.=20Implementation=20details:=20Run=20the=20filte?= =?UTF-8?q?red=20unittest=20discovery=20once.=20Non-goals:=20No=20product?= =?UTF-8?q?=20edits=20here;=20proof=20only.=20Layer:=20app=5Fregression=20?= =?UTF-8?q?Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 964f7535-7395-4bb3-b657-37a77b357f44 From b182d29c691a90c8213d70746a244183ae5f7aba Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:05:38 +0000 Subject: [PATCH 03/13] =?UTF-8?q?invoker:=20wf-1789279240280-10/verify-jud?= =?UTF-8?q?ge-full-suite=20=E2=80=94=20Proof=20step=202=20for=20the=20llm-?= =?UTF-8?q?judge=20investigate=20mode.=20Review=20claim:=20Every=20llm-jud?= =?UTF-8?q?ge=20test=20still=20passes=20after=20the=20change.=20Review=20l?= =?UTF-8?q?ane:=20proof=20Safety=20invariant:=20Proof-only;=20adds=20no=20?= =?UTF-8?q?product=20behavior.=20Effectiveness=20measurement:=20The=20whol?= =?UTF-8?q?e=20llm-judge=20test=20folder=20exits=200.=20Slice=20rationale:?= =?UTF-8?q?=20One=20proof=20step=20guarding=20current=20judge=20callers.?= =?UTF-8?q?=20Architectural=20effect:=20None;=20verification=20only.=20Goa?= =?UTF-8?q?l:=20Prove=20default=20jobs=20behave=20as=20before.=20Motivatio?= =?UTF-8?q?n:=20wrong-check-reflect=20and=20diu-stop=20already=20depend=20?= =?UTF-8?q?on=20the=20judge.=20Alternative=20considerations:=20Running=20o?= =?UTF-8?q?nly=20the=20new=20tests=20was=20rejected;=20it=20would=20miss?= =?UTF-8?q?=20a=20regression=20in=20the=20default=20runner=20set.=20Implem?= =?UTF-8?q?entation=20details:=20Run=20the=20whole=20llm-judge=20test=20fo?= =?UTF-8?q?lder.=20Non-goals:=20No=20product=20edits=20here;=20proof=20onl?= =?UTF-8?q?y.=20Layer:=20app=5Fregression=20Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: c94db032-2d49-4594-b6e7-b3ac43e7b56b From 538767ddc1d37f1edfde973fa05136063ff1006d Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:06:18 +0000 Subject: [PATCH 04/13] =?UTF-8?q?invoker:=20wf-1789279240280-10/verify-jud?= =?UTF-8?q?ge-no-new-comments=20=E2=80=94=20Proof=20step=203=20for=20the?= =?UTF-8?q?=20llm-judge=20investigate=20mode.=20Review=20claim:=20The=20ch?= =?UTF-8?q?ange=20adds=20no=20explanatory=20code=20comments.=20Review=20la?= =?UTF-8?q?ne:=20proof=20Safety=20invariant:=20Proof-only;=20adds=20no=20p?= =?UTF-8?q?roduct=20behavior.=20Effectiveness=20measurement:=20The=20comme?= =?UTF-8?q?nt=20gate=20exits=200=20against=20origin/main.=20Slice=20ration?= =?UTF-8?q?ale:=20One=20proof=20step=20for=20catstack's=20required=20comme?= =?UTF-8?q?nt=20gate.=20Architectural=20effect:=20None;=20verification=20o?= =?UTF-8?q?nly.=20Goal:=20Keep=20the=20CI=20comment=20gate=20green.=20Moti?= =?UTF-8?q?vation:=20CI=20runs=20this=20gate=20on=20every=20pull=20request?= =?UTF-8?q?.=20Alternative=20considerations:=20Waiting=20for=20CI=20was=20?= =?UTF-8?q?rejected;=20it=20is=20slower=20to=20find.=20Implementation=20de?= =?UTF-8?q?tails:=20Run=20the=20comment=20gate=20against=20origin/main.=20?= =?UTF-8?q?Non-goals:=20No=20product=20edits=20here;=20proof=20only.=20Lay?= =?UTF-8?q?er:=20app=5Fregression=20Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 492bf3e7-0cff-46b3-b0d1-89687e47b6b7 From a711d7d25689a13101b5271cd7b1dac471355a6e Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:06:59 +0000 Subject: [PATCH 05/13] =?UTF-8?q?invoker:=20wf-1789279240280-10/scrub-hand?= =?UTF-8?q?off-artifacts=20=E2=80=94=20Check=20that=20no=20ephemeral=20int?= =?UTF-8?q?er-task=20handoff=20files=20remain=20before=20the=20merge=20gat?= =?UTF-8?q?e.=20Review=20claim:=20Only=20ephemeral=20handoff=20files=20are?= =?UTF-8?q?=20checked;=20no=20product=20or=20test=20file=20is=20touched.?= =?UTF-8?q?=20Review=20lane:=20cleanup=20Safety=20invariant:=20The=20scrub?= =?UTF-8?q?=20script=20runs=20read-only=20and=20never=20touches=20the=20ho?= =?UTF-8?q?me=20Invoker=20ledger.=20Effectiveness=20measurement:=20The=20s?= =?UTF-8?q?crub=20script=20exits=200=20and=20reports=20no=20handoff=20path?= =?UTF-8?q?.=20Slice=20rationale:=20One=20cleanup=20slice=20required=20on?= =?UTF-8?q?=20every=20implementation=20plan=20that=20opens=20a=20pull=20re?= =?UTF-8?q?quest,=20kept=20separate=20from=20behavior=20and=20proof=20work?= =?UTF-8?q?.=20Architectural=20effect:=20None.=20Goal:=20Leave=20a=20clean?= =?UTF-8?q?=20worktree=20for=20the=20pull=20request.=20Motivation:=20Ephem?= =?UTF-8?q?eral=20inter-task=20files=20must=20not=20leak=20into=20the=20re?= =?UTF-8?q?viewed=20diff.=20Alternative=20considerations:=20Leaving=20this?= =?UTF-8?q?=20step=20out=20was=20rejected;=20it=20is=20a=20hard=20requirem?= =?UTF-8?q?ent=20for=20every=20plan=20that=20opens=20a=20pull=20request.?= =?UTF-8?q?=20Implementation=20details:=20Run=20the=20repository's=20hando?= =?UTF-8?q?ff-scrub=20script=20without=20--apply.=20Non-goals:=20No=20prod?= =?UTF-8?q?uct=20edits=20in=20this=20task.=20Layer:=20docs=20Feature=20sta?= =?UTF-8?q?te:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 2aafd01f-794a-46f1-8763-0a04ed358458 From 3ca9331edeb7166949b7da32eb611f8e2f20f99d Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:15:46 +0000 Subject: [PATCH 06/13] =?UTF-8?q?invoker:=20wf-1789279268766-11/implement-?= =?UTF-8?q?inbox-report-line=20=E2=80=94=20Append=20the=20judge=20answer's?= =?UTF-8?q?=20report=20string=20to=20a=20hit=20message=20in=20the=20llm-ju?= =?UTF-8?q?dge=20inbox.=20Review=20claim:=20inbox.messages()=20returns=20o?= =?UTF-8?q?n=5Fhit=20followed=20by=20a=20space=20and=20answer["report"]=20?= =?UTF-8?q?when=20the=20verdict=20is=20a=20hit=20and=20report=20is=20a=20n?= =?UTF-8?q?on-empty=20string,=20clipped=20to=20600=20characters;=20every?= =?UTF-8?q?=20other=20verdict=20produces=20the=20same=20text=20as=20before?= =?UTF-8?q?.=20Review=20lane:=20behavior=20Safety=20invariant:=20A=20verdi?= =?UTF-8?q?ct=20whose=20answer=20carries=20no=20"report"=20string=20yields?= =?UTF-8?q?=20byte-for-byte=20the=20same=20message=20as=20today,=20so=20wr?= =?UTF-8?q?ong-check-reflect=20and=20diu-stop=20messages=20are=20unchanged?= =?UTF-8?q?.=20Clean=20verdicts=20still=20produce=20nothing.=20Effectivene?= =?UTF-8?q?ss=20measurement:=20Unit=20tests=20prove=20a=20hit=20with=20a?= =?UTF-8?q?=20report=20ends=20with=20that=20report,=20a=20hit=20lacking=20?= =?UTF-8?q?one=20equals=20on=5Fhit=20exactly,=20a=20report=20over=20600=20?= =?UTF-8?q?characters=20is=20clipped,=20and=20a=20non-string=20report=20is?= =?UTF-8?q?=20ignored.=20Slice=20rationale:=20One=20behavior=20slice=20in?= =?UTF-8?q?=20the=20inbox,=20dormant=20because=20no=20current=20judge=20pr?= =?UTF-8?q?ompt=20asks=20for=20a=20report=20key;=20the=20hook=20that=20ask?= =?UTF-8?q?s=20for=20it=20lands=20next.=20Architectural=20effect:=20The=20?= =?UTF-8?q?inbox=20can=20carry=20per-verdict=20evidence=20text=20from=20th?= =?UTF-8?q?e=20model=20to=20the=20agent.=20Goal:=20Let=20a=20judge=20tell?= =?UTF-8?q?=20the=20agent=20what=20it=20found,=20not=20just=20that=20it=20?= =?UTF-8?q?found=20something.=20Motivation:=20The=20planned=20unverified-t?= =?UTF-8?q?ag=20check=20must=20report=20whether=20a=20"cannot=20verify"=20?= =?UTF-8?q?blocker=20and=20its=20claim=20were=20true=20or=20false,=20with?= =?UTF-8?q?=20the=20file=20or=20quote=20that=20shows=20it.=20A=20fixed=20o?= =?UTF-8?q?n=5Fhit=20sentence=20cannot=20carry=20that.=20Alternative=20con?= =?UTF-8?q?siderations:=20Building=20the=20message=20inside=20the=20hook?= =?UTF-8?q?=20at=20enqueue=20time=20was=20rejected=20because=20the=20answe?= =?UTF-8?q?r=20does=20not=20exist=20until=20the=20background=20run=20ends.?= =?UTF-8?q?=20A=20new=20verdict=20field=20was=20rejected=20as=20more=20sur?= =?UTF-8?q?face=20than=20one=20optional=20answer=20key.=20Implementation?= =?UTF-8?q?=20details:=20In=20engine/hooks/llm-judge/inbox.py,=20in=20mess?= =?UTF-8?q?ages(),=20after=20picking=20the=20hit=20text,=20read=20item.get?= =?UTF-8?q?("answer");=20when=20it=20is=20a=20dict=20whose=20"report"=20va?= =?UTF-8?q?lue=20is=20a=20str=20with=20non-blank=20content,=20append=20"?= =?UTF-8?q?=20"=20plus=20that=20value=20stripped=20and=20clipped=20to=20RE?= =?UTF-8?q?PORT=5FLIMIT=20=3D=20600=20characters.=20Add=20tests=20to=20eng?= =?UTF-8?q?ine/hooks/llm-judge/tests/test=5Finbox.py.=20Non-goals:=20No=20?= =?UTF-8?q?change=20to=20judge.py,=20to=20unchecked=20or=20clean=20handlin?= =?UTF-8?q?g,=20or=20to=20any=20hook.=20No=20README=20edit=20in=20this=20t?= =?UTF-8?q?ask.=20Layer:=20domain=20Feature=20state:=20dormant=20Files:=20?= =?UTF-8?q?-=20engine/hooks/llm-judge/inbox.py=20-=20engine/hooks/llm-judg?= =?UTF-8?q?e/tests/test=5Finbox.py=20Change=20types:=20-=20engine/hooks/ll?= =?UTF-8?q?m-judge/inbox.py:=20modify=20-=20engine/hooks/llm-judge/tests/t?= =?UTF-8?q?est=5Finbox.py:=20modify=20Acceptance=20criteria:=20-=20Tests?= =?UTF-8?q?=20whose=20names=20contain=20"report"=20pass.=20-=20The=20whole?= =?UTF-8?q?=20llm-judge=20test=20folder=20passes.=20-=20python3=20scripts/?= =?UTF-8?q?check=5Fno=5Fnew=5Fcomments.py=20--base=20origin/main=20exits?= =?UTF-8?q?=200.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution: Append the judge answer's report string to a hit message in the llm-judge inbox. Review claim: inbox.messages() returns on_hit followed by a space and answer["report"] when the verdict is a hit and report is a non-empty string, clipped to 600 characters; every other verdict produces the same text as before. Review lane: behavior Safety invariant: A verdict whose answer carries no "report" string yields byte-for-byte the same message as today, so wrong-check-reflect and diu-stop messages are unchanged. Clean verdicts still produce nothing. Effectiveness measurement: Unit tests prove a hit with a report ends with that report, a hit lacking one equals on_hit exactly, a report over 600 characters is clipped, and a non-string report is ignored. Slice rationale: One behavior slice in the inbox, dormant because no current judge prompt asks for a report key; the hook that asks for it lands next. Architectural effect: The inbox can carry per-verdict evidence text from the model to the agent. Goal: Let a judge tell the agent what it found, not just that it found something. Motivation: The planned unverified-tag check must report whether a "cannot verify" blocker and its claim were true or false, with the file or quote that shows it. A fixed on_hit sentence cannot carry that. Alternative considerations: Building the message inside the hook at enqueue time was rejected because the answer does not exist until the background run ends. A new verdict field was rejected as more surface than one optional answer key. Implementation details: In engine/hooks/llm-judge/inbox.py, in messages(), after picking the hit text, read item.get("answer"); when it is a dict whose "report" value is a str with non-blank content, append " " plus that value stripped and clipped to REPORT_LIMIT = 600 characters. Add tests to engine/hooks/llm-judge/tests/test_inbox.py. Non-goals: No change to judge.py, to unchecked or clean handling, or to any hook. No README edit in this task. Layer: domain Feature state: dormant Files: - engine/hooks/llm-judge/inbox.py - engine/hooks/llm-judge/tests/test_inbox.py Change types: - engine/hooks/llm-judge/inbox.py: modify - engine/hooks/llm-judge/tests/test_inbox.py: modify Acceptance criteria: - Tests whose names contain "report" pass. - The whole llm-judge test folder passes. - python3 scripts/check_no_new_comments.py --base origin/main exits 0. Invoker-Finalize-Id: c48ed200-f1d8-4426-b7f9-e0b2f09a285c --- engine/hooks/llm-judge/inbox.py | 6 +++++ engine/hooks/llm-judge/tests/test_inbox.py | 28 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/engine/hooks/llm-judge/inbox.py b/engine/hooks/llm-judge/inbox.py index fd9766fa..ed285123 100644 --- a/engine/hooks/llm-judge/inbox.py +++ b/engine/hooks/llm-judge/inbox.py @@ -6,6 +6,7 @@ import judge NO_TRANSCRIPT = "llm-judge: {harness} payload has no transcript path, so finished verdicts were not checked" +REPORT_LIMIT = 600 def resolve_transcript(payload: dict) -> str: @@ -49,6 +50,11 @@ def messages(transcript: str) -> list[str]: text = item.get("on_hit") if not isinstance(text, str) or not text.strip(): text = f"llm-judge: {item.get('hook') or 'unknown hook'} flagged the last reply: {item.get('reason')}" + answer = item.get("answer") + if isinstance(answer, dict): + report = answer.get("report") + if isinstance(report, str) and report.strip(): + text = f"{text} {report.strip()[:REPORT_LIMIT]}" out.append(text) continue out.append(unchecked_message(item)) diff --git a/engine/hooks/llm-judge/tests/test_inbox.py b/engine/hooks/llm-judge/tests/test_inbox.py index b142bdcb..af270bd4 100644 --- a/engine/hooks/llm-judge/tests/test_inbox.py +++ b/engine/hooks/llm-judge/tests/test_inbox.py @@ -56,6 +56,9 @@ def seed(self, *runner_entries, job_id="job-1"): }) return judge.run_job(job_path) + def seed_verdict(self, verdict, job_id="job-1"): + judge.write_json_atomic(os.path.join(judge.verdict_dir(self.transcript), f"{job_id}.json"), verdict) + def run_claude(self, stdin_text): out, err = io.StringIO(), io.StringIO() with patch.object(sys, "stdin", io.StringIO(stdin_text)), redirect_stdout(out), redirect_stderr(err): @@ -84,6 +87,31 @@ def test_hit_yields_the_exact_on_hit_text_once(self): self.assertEqual(inbox.messages(self.transcript), [ON_HIT]) self.assertEqual(inbox.messages(self.transcript), []) + def test_hit_with_report_appends_report(self): + report = "model saw a quoted rollback" + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}}) + found = inbox.messages(self.transcript) + self.assertEqual(found, [f"{ON_HIT} {report}"]) + self.assertTrue(found[0].endswith(f" {report}"), found) + + def test_hit_without_report_equals_on_hit_exactly(self): + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"match": True}}) + self.assertEqual(inbox.messages(self.transcript), [ON_HIT]) + + def test_hit_report_is_clipped_to_600_characters(self): + report = "x" * 700 + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": report}}) + self.assertEqual(inbox.messages(self.transcript), [f"{ON_HIT} {'x' * 600}"]) + + def test_hit_number_or_list_report_is_ignored(self): + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": 5}}, job_id="a") + self.seed_verdict({"outcome": "hit", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": ["detail"]}}, job_id="b") + self.assertEqual(inbox.messages(self.transcript), [ON_HIT, ON_HIT]) + + def test_clean_with_report_yields_nothing(self): + self.seed_verdict({"outcome": "clean", "hook": "demo-hook", "on_hit": ON_HIT, "answer": {"report": "ignored"}}) + self.assertEqual(inbox.messages(self.transcript), []) + def test_unchecked_yields_one_reason_per_runner(self): self.assertEqual(self.seed(MISSING, CRASHES)["outcome"], "unchecked") self.assertEqual( From f12cf3241a8ca1c957ec0bd1a0cf7f8aa3f08c4b Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:16:32 +0000 Subject: [PATCH 07/13] =?UTF-8?q?invoker:=20wf-1789279268766-11/verify-inb?= =?UTF-8?q?ox-report-tests=20=E2=80=94=20Proof=20step=201=20for=20the=20in?= =?UTF-8?q?box=20report=20line.=20Review=20claim:=20The=20report-line=20un?= =?UTF-8?q?it=20tests=20run=20and=20pass.=20Review=20lane:=20proof=20Safet?= =?UTF-8?q?y=20invariant:=20Proof-only;=20adds=20no=20product=20behavior.?= =?UTF-8?q?=20Effectiveness=20measurement:=20The=20run=20exits=200=20and?= =?UTF-8?q?=20reports=20at=20least=20one=20test;=20Python=20exits=205=20wh?= =?UTF-8?q?en=20-k=20matches=20nothing.=20Slice=20rationale:=20One=20proof?= =?UTF-8?q?=20step=20for=20this=20workflow's=20single=20change.=20Architec?= =?UTF-8?q?tural=20effect:=20None;=20verification=20only.=20Goal:=20Prove?= =?UTF-8?q?=20the=20report=20line=20deterministically.=20Motivation:=20Eac?= =?UTF-8?q?h=20slice=20carries=20its=20own=20executable=20proof.=20Alterna?= =?UTF-8?q?tive=20considerations:=20Manual=20inspection=20was=20rejected?= =?UTF-8?q?=20as=20non-deterministic.=20Implementation=20details:=20Run=20?= =?UTF-8?q?the=20filtered=20unittest=20discovery=20once.=20Non-goals:=20No?= =?UTF-8?q?=20product=20edits=20here;=20proof=20only.=20Layer:=20app=5Freg?= =?UTF-8?q?ression=20Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 7ffa5643-18c9-4676-875e-a4105c1be37b From 34ddc997fbb4c1355118202a448269cbaf45dd80 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:17:21 +0000 Subject: [PATCH 08/13] =?UTF-8?q?invoker:=20wf-1789279268766-11/verify-inb?= =?UTF-8?q?ox-full-suite=20=E2=80=94=20Proof=20step=202=20for=20the=20inbo?= =?UTF-8?q?x=20report=20line.=20Review=20claim:=20Every=20llm-judge=20test?= =?UTF-8?q?=20still=20passes=20after=20the=20change.=20Review=20lane:=20pr?= =?UTF-8?q?oof=20Safety=20invariant:=20Proof-only;=20adds=20no=20product?= =?UTF-8?q?=20behavior.=20Effectiveness=20measurement:=20The=20whole=20llm?= =?UTF-8?q?-judge=20test=20folder=20exits=200.=20Slice=20rationale:=20One?= =?UTF-8?q?=20proof=20step=20guarding=20current=20inbox=20callers.=20Archi?= =?UTF-8?q?tectural=20effect:=20None;=20verification=20only.=20Goal:=20Pro?= =?UTF-8?q?ve=20messages=20for=20current=20callers=20are=20unchanged.=20Mo?= =?UTF-8?q?tivation:=20wrong-check-reflect=20and=20diu-stop=20already=20de?= =?UTF-8?q?liver=20through=20the=20inbox.=20Alternative=20considerations:?= =?UTF-8?q?=20Running=20only=20the=20new=20tests=20was=20rejected;=20it=20?= =?UTF-8?q?would=20miss=20a=20regression=20for=20current=20callers.=20Impl?= =?UTF-8?q?ementation=20details:=20Run=20the=20whole=20llm-judge=20test=20?= =?UTF-8?q?folder.=20Non-goals:=20No=20product=20edits=20here;=20proof=20o?= =?UTF-8?q?nly.=20Layer:=20app=5Fregression=20Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 4c6eee23-2412-44e4-bb8e-665509f2f540 From 80c506445abde842938dd3fd418a41c8ba453a15 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:18:04 +0000 Subject: [PATCH 09/13] =?UTF-8?q?invoker:=20wf-1789279268766-11/verify-inb?= =?UTF-8?q?ox-no-new-comments=20=E2=80=94=20Proof=20step=203=20for=20the?= =?UTF-8?q?=20inbox=20report=20line.=20Review=20claim:=20The=20change=20ad?= =?UTF-8?q?ds=20no=20explanatory=20code=20comments.=20Review=20lane:=20pro?= =?UTF-8?q?of=20Safety=20invariant:=20Proof-only;=20adds=20no=20product=20?= =?UTF-8?q?behavior.=20Effectiveness=20measurement:=20The=20comment=20gate?= =?UTF-8?q?=20exits=200=20against=20origin/main.=20Slice=20rationale:=20On?= =?UTF-8?q?e=20proof=20step=20for=20catstack's=20required=20comment=20gate?= =?UTF-8?q?.=20Architectural=20effect:=20None;=20verification=20only.=20Go?= =?UTF-8?q?al:=20Keep=20the=20CI=20comment=20gate=20green.=20Motivation:?= =?UTF-8?q?=20CI=20runs=20this=20gate=20on=20every=20pull=20request.=20Alt?= =?UTF-8?q?ernative=20considerations:=20Waiting=20for=20CI=20was=20rejecte?= =?UTF-8?q?d;=20it=20is=20slower=20to=20find.=20Implementation=20details:?= =?UTF-8?q?=20Run=20the=20comment=20gate=20against=20origin/main.=20Non-go?= =?UTF-8?q?als:=20No=20product=20edits=20here;=20proof=20only.=20Layer:=20?= =?UTF-8?q?app=5Fregression=20Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 49853f19-8166-4bc3-89a1-dbf73ed9688c From e3d91c9505ed68ebabcb8d28aea88aa9b4fef48e Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:16:57 +0000 Subject: [PATCH 10/13] =?UTF-8?q?invoker:=20wf-1789279268766-11/document-i?= =?UTF-8?q?nbox-report-line=20=E2=80=94=20Describe=20the=20report=20line?= =?UTF-8?q?=20in=20the=20llm-judge=20README's=20inbox=20section.=20Review?= =?UTF-8?q?=20claim:=20The=20"Delivery:=20the=20inbox"=20section=20of=20en?= =?UTF-8?q?gine/hooks/llm-judge/README.md=20says=20a=20hit=20whose=20answe?= =?UTF-8?q?r=20carries=20a=20report=20string=20ends=20with=20that=20report?= =?UTF-8?q?,=20clipped=20to=20600=20characters.=20Review=20lane:=20docs=20?= =?UTF-8?q?Safety=20invariant:=20Only=20engine/hooks/llm-judge/README.md?= =?UTF-8?q?=20changes;=20no=20code=20or=20test=20file=20is=20edited.=20Eff?= =?UTF-8?q?ectiveness=20measurement:=20grep=20finds=20the=20word=20report?= =?UTF-8?q?=20and=20the=20number=20600=20in=20the=20README=20inbox=20secti?= =?UTF-8?q?on.=20Slice=20rationale:=20The=20README=20line=20for=20the=20be?= =?UTF-8?q?havior=20in=20this=20same=20pull=20request,=20which=20catstack'?= =?UTF-8?q?s=20ship-a-detector=20playbook=20requires=20to=20ship=20togethe?= =?UTF-8?q?r.=20Architectural=20effect:=20None;=20README=20only.=20Goal:?= =?UTF-8?q?=20Keep=20the=20inbox=20contract=20in=20the=20README=20accurate?= =?UTF-8?q?.=20Motivation:=20catstack=20keeps=20every=20hook's=20contract?= =?UTF-8?q?=20in=20its=20README.=20Alternative=20considerations:=20A=20lat?= =?UTF-8?q?er=20README-only=20pull=20request=20was=20rejected;=20the=20pla?= =?UTF-8?q?ybook=20counts=20that=20as=20a=20known=20repair=20pattern.=20Im?= =?UTF-8?q?plementation=20details:=20Edit=20the=20hit=20bullet=20under=20"?= =?UTF-8?q?##=20Delivery:=20the=20inbox"=20in=20engine/hooks/llm-judge/REA?= =?UTF-8?q?DME.md.=20Non-goals:=20No=20code=20or=20test=20edits.=20Layer:?= =?UTF-8?q?=20docs=20Feature=20state:=20active=20Files:=20-=20engine/hooks?= =?UTF-8?q?/llm-judge/README.md=20Change=20types:=20-=20engine/hooks/llm-j?= =?UTF-8?q?udge/README.md:=20modify=20Acceptance=20criteria:=20-=20`grep?= =?UTF-8?q?=20-n=20"report"=20engine/hooks/llm-judge/README.md`=20exits=20?= =?UTF-8?q?0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context: wf-1789279268766-11/implement-inbox-report-line (0d98bfa): Append the judge answer's report string to a hit message in the llm-judge inbox. Review claim: inbox.messages() returns on_hit followed by a space and answer["report"] when the verdict is a hit and report is a non-empty string, clipped to 600 characters; every other verdict produces the same text as before. Review lane: behavior Safety invariant: A verdict whose answer carries no "report" string yields byte-for-byte the same message as today, so wrong-check-reflect and diu-stop messages are unchanged. Clean verdicts still produce nothing. Effectiveness measurement: Unit tests prove a hit with a report ends with that report, a hit lacking one equals on_hit exactly, a report over 600 characters is clipped, and a non-string report is ignored. Slice rationale: One behavior slice in the inbox, dormant because no current judge prompt asks for a report key; the hook that asks for it lands next. Architectural effect: The inbox can carry per-verdict evidence text from the model to the agent. Goal: Let a judge tell the agent what it found, not just that it found something. Motivation: The planned unverified-tag check must report whether a "cannot verify" blocker and its claim were true or false, with the file or quote that shows it. A fixed on_hit sentence cannot carry that. Alternative considerations: Building the message inside the hook at enqueue time was rejected because the answer does not exist until the background run ends. A new verdict field was rejected as more surface than one optional answer key. Implementation details: In engine/hooks/llm-judge/inbox.py, in messages(), after picking the hit text, read item.get("answer"); when it is a dict whose "report" value is a str with non-blank content, append " " plus that value stripped and clipped to REPORT_LIMIT = 600 characters. Add tests to engine/hooks/llm-judge/tests/test_inbox.py. Non-goals: No change to judge.py, to unchecked or clean handling, or to any hook. No README edit in this task. Layer: domain Feature state: dormant Files: - engine/hooks/llm-judge/inbox.py - engine/hooks/llm-judge/tests/test_inbox.py Change types: - engine/hooks/llm-judge/inbox.py: modify - engine/hooks/llm-judge/tests/test_inbox.py: modify Acceptance criteria: - Tests whose names contain "report" pass. - The whole llm-judge test folder passes. - python3 scripts/check_no_new_comments.py --base origin/main exits 0. Solution: Describe the report line in the llm-judge README's inbox section. Review claim: The "Delivery: the inbox" section of engine/hooks/llm-judge/README.md says a hit whose answer carries a report string ends with that report, clipped to 600 characters. Review lane: docs Safety invariant: Only engine/hooks/llm-judge/README.md changes; no code or test file is edited. Effectiveness measurement: grep finds the word report and the number 600 in the README inbox section. Slice rationale: The README line for the behavior in this same pull request, which catstack's ship-a-detector playbook requires to ship together. Architectural effect: None; README only. Goal: Keep the inbox contract in the README accurate. Motivation: catstack keeps every hook's contract in its README. Alternative considerations: A later README-only pull request was rejected; the playbook counts that as a known repair pattern. Implementation details: Edit the hit bullet under "## Delivery: the inbox" in engine/hooks/llm-judge/README.md. Non-goals: No code or test edits. Layer: docs Feature state: active Files: - engine/hooks/llm-judge/README.md Change types: - engine/hooks/llm-judge/README.md: modify Acceptance criteria: - `grep -n "report" engine/hooks/llm-judge/README.md` exits 0. Invoker-Finalize-Id: c105fd49-67e1-44b6-827f-95db211a0555 --- engine/hooks/llm-judge/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/hooks/llm-judge/README.md b/engine/hooks/llm-judge/README.md index 31eba814..ec80ca2d 100644 --- a/engine/hooks/llm-judge/README.md +++ b/engine/hooks/llm-judge/README.md @@ -165,7 +165,9 @@ hold up the reply. `inbox.messages(transcript)` drains that transcript's verdicts and turns each one into a line of text: -- **hit**: the job's `on_hit` text, word for word. +- **hit**: the job's `on_hit` text, word for word, followed by a space and the + answer's `report` string when the answer has a non-blank one, clipped to 600 + characters. - **unchecked**: `llm-judge: could not judge the last reply: ` then `: ` for each try, joined by `; `. If there were no tries (the judge broke, or the verdict file was unreadable), the verdict's own From ef227f442a7e7aeb5002bf4188bdfa339f4dbafb Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:18:46 +0000 Subject: [PATCH 11/13] =?UTF-8?q?invoker:=20wf-1789279268766-11/scrub-hand?= =?UTF-8?q?off-artifacts=20=E2=80=94=20Check=20that=20no=20ephemeral=20int?= =?UTF-8?q?er-task=20handoff=20files=20remain=20before=20the=20merge=20gat?= =?UTF-8?q?e.=20Review=20claim:=20Only=20ephemeral=20handoff=20files=20are?= =?UTF-8?q?=20checked;=20no=20product=20or=20test=20file=20is=20touched.?= =?UTF-8?q?=20Review=20lane:=20cleanup=20Safety=20invariant:=20The=20scrub?= =?UTF-8?q?=20script=20runs=20read-only=20and=20never=20touches=20the=20ho?= =?UTF-8?q?me=20Invoker=20ledger.=20Effectiveness=20measurement:=20The=20s?= =?UTF-8?q?crub=20script=20exits=200=20and=20reports=20no=20handoff=20path?= =?UTF-8?q?.=20Slice=20rationale:=20One=20cleanup=20slice=20required=20on?= =?UTF-8?q?=20every=20implementation=20plan=20that=20opens=20a=20pull=20re?= =?UTF-8?q?quest,=20kept=20separate=20from=20behavior=20and=20proof=20work?= =?UTF-8?q?.=20Architectural=20effect:=20None.=20Goal:=20Leave=20a=20clean?= =?UTF-8?q?=20worktree=20for=20the=20pull=20request.=20Motivation:=20Ephem?= =?UTF-8?q?eral=20inter-task=20files=20must=20not=20leak=20into=20the=20re?= =?UTF-8?q?viewed=20diff.=20Alternative=20considerations:=20Leaving=20this?= =?UTF-8?q?=20step=20out=20was=20rejected;=20it=20is=20a=20hard=20requirem?= =?UTF-8?q?ent=20for=20every=20plan=20that=20opens=20a=20pull=20request.?= =?UTF-8?q?=20Implementation=20details:=20Run=20the=20repository's=20hando?= =?UTF-8?q?ff-scrub=20script=20without=20--apply.=20Non-goals:=20No=20prod?= =?UTF-8?q?uct=20edits=20in=20this=20task.=20Layer:=20docs=20Feature=20sta?= =?UTF-8?q?te:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 007250b8-44a0-4a48-a256-7fd291266e8f From 2371e0a52a172e1b19c884d46d608f1ec3dc3d1c Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sun, 13 Sep 2026 21:09:34 -0700 Subject: [PATCH 12/13] principle-prove-it: example where the pasted output is narrower than the claim The skill test coverage gate requires a test change with any rule-shaped SKILL.md change. The new entailment rule gets a fires_ example, matching how the subagent-contradiction rule got fires_subagent_contradiction.md. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HQ9TkZeqARxv3e7W7xesYy Change-Id: Ia60c5c6a87b249bb186b83a5be5b0717603536bd --- .../tests/fires_narrow_output_wide_claim.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md diff --git a/corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md b/corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md new file mode 100644 index 00000000..844c4b67 --- /dev/null +++ b/corpus/skills/principle-prove-it/tests/fires_narrow_output_wide_claim.md @@ -0,0 +1,17 @@ +An agent runs `docker run --rm node:25-slim corepack --version` and pastes the +real output: `corepack: not found`. It then writes "verified: corepack was +removed from Node 25+" into the PR body. One image, one tag, one run. Nothing +checked a second 25.x image, a later version, or the release notes. + +This skill fires. The pasted output is real and every word of it is true, but +it does not entail the sentence. It rules out corepack in that one image; it +says nothing about the version boundary the claim draws. The rule that the +output must entail the sentence, not merely agree with it, is what the reply +needs: rewrite the claim down to what ran ("the `node:25-slim` image has no +corepack") and mark the wider statement as open, or run the check that would +actually cover it. + +The same shape fires on a correction. If the agent later says "I overstated +it earlier, here is the proof" and pastes the same single-image run under the +same "removed from Node 25+" heading, the correction has relabeled the +overclaim as a fix. The narrower proof needs the narrower sentence. From 1aaedf3707528e2fabeb1b11fa017752463ff096 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sat, 12 Sep 2026 19:57:02 -0700 Subject: [PATCH 13/13] test(llm-judge): the shared test base runs only a local stub judge Tests on the base can no longer fall through to codex/claude/cursor, and enqueue is pinned to the temporary state folder. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013smxPCr4XeE7R77zVtN1u4 Change-Id: I7b2356d2fda21cbe253900540139827fcc4d8827 --- engine/hooks/llm-judge/judge_test_base.py | 9 +++++++-- engine/hooks/llm-judge/tests/test_judge.py | 22 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/engine/hooks/llm-judge/judge_test_base.py b/engine/hooks/llm-judge/judge_test_base.py index 01249d18..9bb41eb8 100644 --- a/engine/hooks/llm-judge/judge_test_base.py +++ b/engine/hooks/llm-judge/judge_test_base.py @@ -2,6 +2,7 @@ import json import os +import sys import tempfile import unittest from unittest.mock import patch @@ -13,10 +14,14 @@ class JudgeTestCase(unittest.TestCase): def setUp(self): super().setUp() self.state = tempfile.TemporaryDirectory() - self.judge_env = patch.dict(os.environ, {judge.STATE_ENV: self.state.name}) + self.judge_env = patch.dict(os.environ, { + judge.STATE_ENV: self.state.name, + judge.RUNNERS_ENV: json.dumps([ + ["stub", [sys.executable, "-c", "print('{\"match\": false}')", judge.PROMPT_SLOT]], + ]), + }) self.judge_env.start() os.environ.pop(judge.CHILD_ENV, None) - os.environ.pop(judge.RUNNERS_ENV, None) def tearDown(self): self.judge_env.stop() diff --git a/engine/hooks/llm-judge/tests/test_judge.py b/engine/hooks/llm-judge/tests/test_judge.py index 52246b69..a41cf24d 100644 --- a/engine/hooks/llm-judge/tests/test_judge.py +++ b/engine/hooks/llm-judge/tests/test_judge.py @@ -108,10 +108,17 @@ def test_malformed_runners_env_refuses_instead_of_running_defaults(self): with self.assertRaises(ValueError): judge.ask("x") + def test_test_base_runs_only_the_local_stub(self): + self.assertEqual([name for name, _ in judge.runners()], ["stub"]) + self.assertEqual(judge.ask("x")["answer"], {"match": False}) + def test_default_runner_order_is_codex_then_claude_then_cursor(self): - self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) + with patch.dict(os.environ): + os.environ.pop(judge.RUNNERS_ENV) + self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) def test_investigate_runner_argv_is_read_only_and_excludes_cursor(self): + os.environ.pop(judge.RUNNERS_ENV) self.assertEqual( judge.runners("investigate"), [ @@ -159,6 +166,7 @@ def test_investigate_runners_env_replaces_investigate_defaults(self): self.assertEqual(judge.runners("investigate"), [("probe", custom[1])]) def test_investigate_job_threads_timeout_and_cwd_to_runner(self): + os.environ.pop(judge.RUNNERS_ENV) path = os.path.join(self.state.name, "jobs", "investigate-job.json") with tempfile.TemporaryDirectory() as cwd: judge.write_json_atomic(path, self.job(id="investigate-job", mode="investigate", timeout_seconds=123, cwd=cwd)) @@ -220,6 +228,18 @@ def test_unchecked_when_ask_was_unchecked(self): class TestBackground(JudgeBehaviorTestCase): + def test_enqueue_writes_only_to_temporary_state_directory(self): + with tempfile.TemporaryDirectory() as home: + with patch.dict(os.environ, {"HOME": home}): + with patch.dict(os.environ): + os.environ.pop(judge.STATE_ENV) + default_state = judge.state_root() + os.makedirs(default_state) + with patch.object(judge.subprocess, "Popen"): + self.assertEqual(judge.enqueue(self.job(id="isolated-job")), "isolated-job") + self.assertTrue(os.path.isfile(os.path.join(self.state.name, "jobs", "isolated-job.json"))) + self.assertEqual(os.listdir(default_state), []) + def test_enqueue_as_judge_child_returns_none_and_starts_nothing(self): os.environ[judge.CHILD_ENV] = "1" self.use_runners(ANSWER_MATCH)