From f2435ab7e581130a870ecc0dfde53d7dca912b0b Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:04:17 +0000 Subject: [PATCH 1/6] =?UTF-8?q?invoker:=20wf-1789279240280-10/implement-ju?= =?UTF-8?q?dge-investigate-mode=20=E2=80=94=20Let=20an=20llm-judge=20job?= =?UTF-8?q?=20opt=20into=20a=20read-only=20investigate=20mode=20with=20its?= =?UTF-8?q?=20own=20time=20limit=20and=20working=20folder.=20Review=20clai?= =?UTF-8?q?m:=20A=20job=20carrying=20"mode":=20"investigate"=20runs=20on?= =?UTF-8?q?=20a=20second=20runner=20set=20whose=20Claude=20argv=20grants?= =?UTF-8?q?=20only=20the=20Read,=20Grep=20and=20Glob=20tools=20and=20forbi?= =?UTF-8?q?ds=20Write,=20Edit,=20NotebookEdit=20and=20Bash,=20with=20the?= =?UTF-8?q?=20job's=20own=20timeout=5Fseconds=20and=20cwd;=20a=20job=20lac?= =?UTF-8?q?king=20that=20key=20runs=20exactly=20as=20today.=20Review=20lan?= =?UTF-8?q?e:=20behavior=20Safety=20invariant:=20An=20investigate=20job=20?= =?UTF-8?q?never=20receives=20a=20tool=20that=20can=20write,=20edit,=20or?= =?UTF-8?q?=20execute=20shell=20text.=20The=20Claude=20runner=20argv=20alw?= =?UTF-8?q?ays=20carries=20--allowedTools=20Read=20Grep=20Glob=20and=20--d?= =?UTF-8?q?isallowedTools=20Write=20Edit=20NotebookEdit=20Bash,=20the=20Co?= =?UTF-8?q?dex=20runner=20always=20carries=20--sandbox=20read-only,=20and?= =?UTF-8?q?=20cursor-agent=20is=20never=20used=20for=20investigate=20jobs?= =?UTF-8?q?=20because=20it=20has=20no=20read-only=20switch.=20Effectivenes?= =?UTF-8?q?s=20measurement:=20New=20unit=20tests=20pin=20the=20exact=20arg?= =?UTF-8?q?v=20of=20each=20investigate=20runner=20and=20prove=20a=20defaul?= =?UTF-8?q?t=20job=20still=20gets=2060=20seconds=20in=20an=20empty=20temp?= =?UTF-8?q?=20folder.=20The=20PR=20body=20pastes=20the=20live=20probe=20in?= =?UTF-8?q?=20which=20claude=20-p=20with=20this=20argv=20was=20asked=20to?= =?UTF-8?q?=20Write=20a=20file=20and=20to=20touch=20one=20through=20Bash,?= =?UTF-8?q?=20answered=20{"wrote":false,"read":true,"bash":false},=20and?= =?UTF-8?q?=20neither=20target=20file=20existed=20afterwards.=20Slice=20ra?= =?UTF-8?q?tionale:=20One=20behavior=20slice=20inside=20the=20judge,=20dor?= =?UTF-8?q?mant=20until=20a=20hook=20asks=20for=20an=20investigate=20job,?= =?UTF-8?q?=20so=20a=20reviewer=20can=20judge=20the=20permission=20argv=20?= =?UTF-8?q?on=20its=20own.=20Architectural=20effect:=20llm-judge=20gains?= =?UTF-8?q?=20a=20second,=20read-only=20runner=20set=20chosen=20per=20job;?= =?UTF-8?q?=20the=20default=20runner=20set=20and=20every=20current=20calle?= =?UTF-8?q?r=20stay=20unchanged.=20Goal:=20Give=20later=20hooks=20a=20safe?= =?UTF-8?q?=20way=20to=20ask=20a=20model=20to=20check=20a=20claim=20by=20r?= =?UTF-8?q?eading=20files=20on=20this=20machine.=20Motivation:=20A=20backt?= =?UTF-8?q?est=20of=2044=20unverified=20tags=20from=20the=20last=2014=20da?= =?UTF-8?q?ys=20found=2033=20whose=20stated=20blocker=20was=20false;=20mos?= =?UTF-8?q?t=20could=20have=20been=20checked=20by=20reading=20a=20log,=20a?= =?UTF-8?q?=20transcript,=20or=20a=20source=20file=20already=20on=20disk.?= =?UTF-8?q?=20Today=20each=20judge=20runner=20gets=2060=20seconds=20in=20a?= =?UTF-8?q?n=20empty=20temp=20folder=20with=20no=20tool=20grants,=20so=20i?= =?UTF-8?q?t=20cannot=20read=20anything.=20Alternative=20considerations:?= =?UTF-8?q?=20claude=20--permission-mode=20plan=20was=20rejected=20after?= =?UTF-8?q?=20a=20live=20probe=20wrote=20a=20plan=20file=20under=20~/.clau?= =?UTF-8?q?de/plans.=20A=20Bash=20prefix=20allowlist=20such=20as=20Bash(gi?= =?UTF-8?q?t=20log:*)=20was=20rejected=20after=20a=20live=20probe=20blocke?= =?UTF-8?q?d=20even=20plain=20git=20log,=20so=20shell=20access=20stays=20o?= =?UTF-8?q?ut=20of=20this=20slice.=20Implementation=20details:=20In=20engi?= =?UTF-8?q?ne/hooks/llm-judge/judge.py=20add=20INVESTIGATE=5FRUNNERS=20bes?= =?UTF-8?q?ide=20DEFAULT=5FRUNNERS=20holding=20the=20codex=20and=20claude?= =?UTF-8?q?=20argv=20given=20under=20Runner=20argv.=20Make=20runners()=20t?= =?UTF-8?q?ake=20the=20job=20mode=20and=20pick=20INVESTIGATE=5FRUNNERS=20f?= =?UTF-8?q?or=20"investigate";=20CATSTACK=5FLLM=5FJUDGE=5FRUNNERS=20still?= =?UTF-8?q?=20replaces=20whichever=20set=20is=20picked.=20Make=20run=5Frun?= =?UTF-8?q?ner=20accept=20timeout=5Fseconds=20(default=20TIMEOUT=5FSECONDS?= =?UTF-8?q?,=20capped=20at=20INVESTIGATE=5FTIMEOUT=5FCAP=20=3D=20600)=20an?= =?UTF-8?q?d=20cwd=20(used=20only=20when=20it=20is=20an=20absolute=20path?= =?UTF-8?q?=20to=20a=20folder=20that=20exists,=20else=20the=20temp=20folde?= =?UTF-8?q?r=20plus=20a=20judge.log=20line).=20Thread=20mode,=20timeout=5F?= =?UTF-8?q?seconds=20and=20cwd=20from=20the=20job=20through=20ask()=20and?= =?UTF-8?q?=20run=5Fjob().=20Runner=20argv:=20codex=20exec=20--skip-git-re?= =?UTF-8?q?po-check=20--sandbox=20read-only=20-c=20notify=3D[]=20PROMPT,?= =?UTF-8?q?=20and=20claude=20-p=20--model=20haiku=20--settings=20'{"disabl?= =?UTF-8?q?eAllHooks":=20true}'=20--allowedTools=20Read=20Grep=20Glob=20--?= =?UTF-8?q?disallowedTools=20Write=20Edit=20NotebookEdit=20Bash=20--=20PRO?= =?UTF-8?q?MPT.=20Non-goals:=20No=20hook=20asks=20for=20an=20investigate?= =?UTF-8?q?=20job=20in=20this=20slice.=20No=20inbox=20edit,=20no=20change?= =?UTF-8?q?=20to=20DEFAULT=5FRUNNERS,=20wrong-check-reflect=20or=20diu-sto?= =?UTF-8?q?p.=20No=20shell=20or=20network=20access=20for=20the=20investiga?= =?UTF-8?q?tor.=20Layer:=20domain=20Feature=20state:=20dormant=20Files:=20?= =?UTF-8?q?-=20engine/hooks/llm-judge/judge.py=20-=20engine/hooks/llm-judg?= =?UTF-8?q?e/tests/test=5Fjudge.py=20Change=20types:=20-=20engine/hooks/ll?= =?UTF-8?q?m-judge/judge.py:=20modify=20-=20engine/hooks/llm-judge/tests/t?= =?UTF-8?q?est=5Fjudge.py:=20modify=20Acceptance=20criteria:=20-=20Tests?= =?UTF-8?q?=20whose=20names=20contain=20"investigate"=20pass=20and=20pin?= =?UTF-8?q?=20both=20investigate=20runner=20argv=20exactly.=20-=20A=20test?= =?UTF-8?q?=20proves=20a=20job=20lacking=20a=20mode=20key=20still=20runs?= =?UTF-8?q?=20DEFAULT=5FRUNNERS=20with=20a=2060=20second=20limit.=20-=20A?= =?UTF-8?q?=20test=20proves=20timeout=5Fseconds=20above=20600=20is=20cappe?= =?UTF-8?q?d=20at=20600.=20-=20A=20test=20whose=20name=20contains=20"missi?= =?UTF-8?q?ng"=20proves=20a=20cwd=20that=20does=20not=20exist=20falls=20ba?= =?UTF-8?q?ck=20to=20a=20temp=20folder=20and=20logs=20it.=20-=20python3=20?= =?UTF-8?q?scripts/check=5Fno=5Fnew=5Fcomments.py=20--base=20origin/main?= =?UTF-8?q?=20exits=200.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution: Let an llm-judge job opt into a read-only investigate mode with its own time limit and working folder. Review claim: A job carrying "mode": "investigate" runs on a second runner set whose Claude argv grants only the Read, Grep and Glob tools and forbids Write, Edit, NotebookEdit and Bash, with the job's own timeout_seconds and cwd; a job lacking that key runs exactly as today. Review lane: behavior Safety invariant: An investigate job never receives a tool that can write, edit, or execute shell text. The Claude runner argv always carries --allowedTools Read Grep Glob and --disallowedTools Write Edit NotebookEdit Bash, the Codex runner always carries --sandbox read-only, and cursor-agent is never used for investigate jobs because it has no read-only switch. Effectiveness measurement: New unit tests pin the exact argv of each investigate runner and prove a default job still gets 60 seconds in an empty temp folder. The PR body pastes the live probe in which claude -p with this argv was asked to Write a file and to touch one through Bash, answered {"wrote":false,"read":true,"bash":false}, and neither target file existed afterwards. Slice rationale: One behavior slice inside the judge, dormant until a hook asks for an investigate job, so a reviewer can judge the permission argv on its own. Architectural effect: llm-judge gains a second, read-only runner set chosen per job; the default runner set and every current caller stay unchanged. Goal: Give later hooks a safe way to ask a model to check a claim by reading files on this machine. Motivation: A backtest of 44 unverified tags from the last 14 days found 33 whose stated blocker was false; most could have been checked by reading a log, a transcript, or a source file already on disk. Today each judge runner gets 60 seconds in an empty temp folder with no tool grants, so it cannot read anything. Alternative considerations: claude --permission-mode plan was rejected after a live probe wrote a plan file under ~/.claude/plans. A Bash prefix allowlist such as Bash(git log:*) was rejected after a live probe blocked even plain git log, so shell access stays out of this slice. Implementation details: In engine/hooks/llm-judge/judge.py add INVESTIGATE_RUNNERS beside DEFAULT_RUNNERS holding the codex and claude argv given under Runner argv. Make runners() take the job mode and pick INVESTIGATE_RUNNERS for "investigate"; CATSTACK_LLM_JUDGE_RUNNERS still replaces whichever set is picked. Make run_runner accept timeout_seconds (default TIMEOUT_SECONDS, capped at INVESTIGATE_TIMEOUT_CAP = 600) and cwd (used only when it is an absolute path to a folder that exists, else the temp folder plus a judge.log line). Thread mode, timeout_seconds and cwd from the job through ask() and run_job(). Runner argv: codex exec --skip-git-repo-check --sandbox read-only -c notify=[] PROMPT, and claude -p --model haiku --settings '{"disableAllHooks": true}' --allowedTools Read Grep Glob --disallowedTools Write Edit NotebookEdit Bash -- PROMPT. Non-goals: No hook asks for an investigate job in this slice. No inbox edit, no change to DEFAULT_RUNNERS, wrong-check-reflect or diu-stop. No shell or network access for the investigator. Layer: domain Feature state: dormant Files: - engine/hooks/llm-judge/judge.py - engine/hooks/llm-judge/tests/test_judge.py Change types: - engine/hooks/llm-judge/judge.py: modify - engine/hooks/llm-judge/tests/test_judge.py: modify Acceptance criteria: - Tests whose names contain "investigate" pass and pin both investigate runner argv exactly. - A test proves a job lacking a mode key still runs DEFAULT_RUNNERS with a 60 second limit. - A test proves timeout_seconds above 600 is capped at 600. - A test whose name contains "missing" proves a cwd that does not exist falls back to a temp folder and logs it. - python3 scripts/check_no_new_comments.py --base origin/main exits 0. Invoker-Finalize-Id: 890bf8a3-d647-4273-855b-8ca21b411f55 --- engine/hooks/llm-judge/judge.py | 43 ++++++++--- engine/hooks/llm-judge/tests/test_judge.py | 84 ++++++++++++++++++++++ 2 files changed, 116 insertions(+), 11 deletions(-) diff --git a/engine/hooks/llm-judge/judge.py b/engine/hooks/llm-judge/judge.py index a81f19b7..12696205 100644 --- a/engine/hooks/llm-judge/judge.py +++ b/engine/hooks/llm-judge/judge.py @@ -14,6 +14,7 @@ import uuid TIMEOUT_SECONDS = 60 +INVESTIGATE_TIMEOUT_CAP = 600 KILL_GRACE_SECONDS = 5 REASON_LIMIT = 300 PROMPT_SLOT = "{prompt}" @@ -25,6 +26,10 @@ ("claude", ["claude", "-p", "--model", "haiku", "--settings", '{"disableAllHooks": true}', PROMPT_SLOT]), ("cursor", ["cursor-agent", "-p", "--output-format", "text", PROMPT_SLOT]), ) +INVESTIGATE_RUNNERS = ( + ("codex", ["codex", "exec", "--skip-git-repo-check", "--sandbox", "read-only", "-c", "notify=[]", PROMPT_SLOT]), + ("claude", ["claude", "-p", "--model", "haiku", "--settings", '{"disableAllHooks": true}', "--allowedTools", "Read", "Grep", "Glob", "--disallowedTools", "Write", "Edit", "NotebookEdit", "Bash", "--", PROMPT_SLOT]), +) def state_root() -> str: @@ -49,10 +54,11 @@ def valid_runner(entry: object) -> bool: ) -def runners() -> list[tuple[str, list[str]]]: +def runners(mode: object = None) -> list[tuple[str, list[str]]]: + default = INVESTIGATE_RUNNERS if mode == "investigate" else DEFAULT_RUNNERS raw = os.environ.get(RUNNERS_ENV) if not raw: - return [(name, list(argv)) for name, argv in DEFAULT_RUNNERS] + return [(name, list(argv)) for name, argv in default] try: parsed = json.loads(raw) except ValueError as exc: @@ -96,13 +102,26 @@ def stop_group(proc: subprocess.Popen) -> str: return stderr or "" -def run_runner(name: str, argv: list[str], prompt: str) -> tuple[dict, dict | None]: +def bounded_timeout(timeout_seconds: object) -> int | float: + if isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, (int, float)): + timeout_seconds = TIMEOUT_SECONDS + return min(timeout_seconds, INVESTIGATE_TIMEOUT_CAP) + + +def run_runner(name: str, argv: list[str], prompt: str, timeout_seconds: object = TIMEOUT_SECONDS, cwd: object = None) -> tuple[dict, dict | None]: if shutil.which(argv[0]) is None: return failed(name, "not installed"), None command = [prompt if item == PROMPT_SLOT else item for item in argv] env = dict(os.environ) env[CHILD_ENV] = "1" - with tempfile.TemporaryDirectory(prefix="llm-judge-") as cwd: + timeout = bounded_timeout(timeout_seconds) + with tempfile.TemporaryDirectory(prefix="llm-judge-") as temp_cwd: + runner_cwd = temp_cwd + if cwd is not None: + if isinstance(cwd, str) and os.path.isabs(cwd) and os.path.isdir(cwd): + runner_cwd = cwd + else: + log(f"runner {name}: refused cwd {cwd!r}") try: proc = subprocess.Popen( command, @@ -110,16 +129,16 @@ def run_runner(name: str, argv: list[str], prompt: str) -> tuple[dict, dict | No stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - cwd=cwd, + cwd=runner_cwd, env=env, start_new_session=True, ) except OSError as exc: return failed(name, clip(type(exc).__name__, str(exc))), None try: - stdout, stderr = proc.communicate(timeout=TIMEOUT_SECONDS) + stdout, stderr = proc.communicate(timeout=timeout) except subprocess.TimeoutExpired: - return failed(name, clip(f"timed out after {TIMEOUT_SECONDS}s", stop_group(proc))), None + return failed(name, clip(f"timed out after {timeout}s", stop_group(proc))), None if proc.returncode != 0: return failed(name, clip(f"exit {proc.returncode}", stderr)), None answer = last_json_object(stdout) @@ -128,10 +147,12 @@ def run_runner(name: str, argv: list[str], prompt: str) -> tuple[dict, dict | No return {"runner": name, "ok": True, "reason": "answered"}, answer -def ask(prompt: str) -> dict: +def ask(prompt: str, mode: object = None, timeout_seconds: object = None, cwd: object = None) -> dict: + if timeout_seconds is None: + timeout_seconds = TIMEOUT_SECONDS attempts = [] - for name, argv in runners(): - attempt, answer = run_runner(name, argv, prompt) + for name, argv in runners(mode): + attempt, answer = run_runner(name, argv, prompt, timeout_seconds=timeout_seconds, cwd=cwd) attempts.append(attempt) if answer is not None: return {"outcome": "answered", "runner": name, "answer": answer, "attempts": attempts} @@ -216,7 +237,7 @@ def run_job(path: str) -> dict: raise ValueError(f"job file holds a JSON {type(loaded).__name__}, not an object") job = dict(loaded) job.setdefault("id", stem) - result = verdict(job, ask(str(job["prompt"]))) + result = verdict(job, ask(str(job["prompt"]), mode=job.get("mode"), timeout_seconds=job.get("timeout_seconds", TIMEOUT_SECONDS), cwd=job.get("cwd"))) except Exception as exc: print(f"catstack-hook-error llm-judge: {type(exc).__name__}: {exc}", file=sys.stderr) log(f"job {job.get('id')} failed: {type(exc).__name__}: {exc}\n{traceback.format_exc()}") diff --git a/engine/hooks/llm-judge/tests/test_judge.py b/engine/hooks/llm-judge/tests/test_judge.py index 08d61c0d..8465f0b2 100644 --- a/engine/hooks/llm-judge/tests/test_judge.py +++ b/engine/hooks/llm-judge/tests/test_judge.py @@ -124,6 +124,90 @@ def test_malformed_runners_env_refuses_instead_of_running_defaults(self): def test_default_runner_order_is_codex_then_claude_then_cursor(self): self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) + def test_investigate_runner_argv_is_read_only_and_excludes_cursor(self): + self.assertEqual( + judge.runners("investigate"), + [ + ( + "codex", + [ + "codex", + "exec", + "--skip-git-repo-check", + "--sandbox", + "read-only", + "-c", + "notify=[]", + judge.PROMPT_SLOT, + ], + ), + ( + "claude", + [ + "claude", + "-p", + "--model", + "haiku", + "--settings", + '{"disableAllHooks": true}', + "--allowedTools", + "Read", + "Grep", + "Glob", + "--disallowedTools", + "Write", + "Edit", + "NotebookEdit", + "Bash", + "--", + judge.PROMPT_SLOT, + ], + ), + ], + ) + + def test_investigate_runners_env_replaces_investigate_defaults(self): + custom = ["probe", [PY, "-c", "print('{}')", "{prompt}"]] + self.use_runners(custom) + self.assertEqual(judge.runners("investigate"), [("probe", custom[1])]) + + def test_investigate_job_threads_timeout_and_cwd_to_runner(self): + path = os.path.join(self.state.name, "jobs", "investigate-job.json") + with tempfile.TemporaryDirectory() as cwd: + judge.write_json_atomic(path, self.job(id="investigate-job", mode="investigate", timeout_seconds=123, cwd=cwd)) + calls = [] + + def capture(name, argv, prompt, timeout_seconds=judge.TIMEOUT_SECONDS, cwd=None): + calls.append((name, timeout_seconds, cwd)) + return {"runner": name, "ok": True, "reason": "answered"}, {"match": True} + + with patch.object(judge, "run_runner", side_effect=capture): + result = judge.run_job(path) + + self.assertEqual(result["outcome"], "hit") + self.assertEqual(calls, [("codex", 123, cwd)]) + + def test_investigate_timeout_is_capped_at_600_seconds(self): + self.assertEqual(judge.bounded_timeout(999), 600) + + def test_non_investigate_job_still_gets_default_timeout_and_empty_temp_cwd(self): + self.use_runners(runner("env", "import json, os; print(json.dumps({'cwd': os.getcwd(), 'entries': os.listdir('.')}))")) + path = os.path.join(self.state.name, "jobs", "default-job.json") + judge.write_json_atomic(path, self.job(id="default-job")) + calls = [] + original = judge.run_runner + + def capture(name, argv, prompt, timeout_seconds=judge.TIMEOUT_SECONDS, cwd=None): + calls.append((timeout_seconds, cwd)) + return original(name, argv, prompt, timeout_seconds=timeout_seconds, cwd=cwd) + + with patch.object(judge, "run_runner", side_effect=capture): + result = judge.run_job(path) + + self.assertEqual(calls, [(judge.TIMEOUT_SECONDS, None)]) + self.assertEqual(result["answer"]["entries"], []) + self.assertFalse(os.path.exists(result["answer"]["cwd"])) + class TestVerdict(JudgeTestCase): def test_hit_when_every_hit_key_is_true(self): From 06053d2205c59605a67687e87b6831effbda81aa Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:04:58 +0000 Subject: [PATCH 2/6] =?UTF-8?q?invoker:=20wf-1789279240280-10/verify-judge?= =?UTF-8?q?-investigate-tests=20=E2=80=94=20Proof=20step=201=20for=20the?= =?UTF-8?q?=20llm-judge=20investigate=20mode.=20Review=20claim:=20The=20in?= =?UTF-8?q?vestigate-mode=20unit=20tests=20run=20and=20pass.=20Review=20la?= =?UTF-8?q?ne:=20proof=20Safety=20invariant:=20Proof-only;=20adds=20no=20p?= =?UTF-8?q?roduct=20behavior.=20Effectiveness=20measurement:=20The=20run?= =?UTF-8?q?=20exits=200=20and=20reports=20at=20least=20one=20test;=20Pytho?= =?UTF-8?q?n=20exits=205=20when=20-k=20matches=20nothing.=20Slice=20ration?= =?UTF-8?q?ale:=20One=20proof=20step=20for=20this=20workflow's=20single=20?= =?UTF-8?q?change.=20Architectural=20effect:=20None;=20verification=20only?= =?UTF-8?q?.=20Goal:=20Prove=20the=20investigate=20runner=20argv=20and=20t?= =?UTF-8?q?ime=20limit=20deterministically.=20Motivation:=20Each=20slice?= =?UTF-8?q?=20carries=20its=20own=20executable=20proof.=20Alternative=20co?= =?UTF-8?q?nsiderations:=20Manual=20inspection=20was=20rejected=20as=20non?= =?UTF-8?q?-deterministic.=20Implementation=20details:=20Run=20the=20filte?= =?UTF-8?q?red=20unittest=20discovery=20once.=20Non-goals:=20No=20product?= =?UTF-8?q?=20edits=20here;=20proof=20only.=20Layer:=20app=5Fregression=20?= =?UTF-8?q?Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 964f7535-7395-4bb3-b657-37a77b357f44 From 361358e396a2c4b124360db295ad35e856b5ecb7 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:05:27 +0000 Subject: [PATCH 3/6] =?UTF-8?q?invoker:=20wf-1789279240280-10/document-jud?= =?UTF-8?q?ge-investigate-mode=20=E2=80=94=20Describe=20the=20llm-judge=20?= =?UTF-8?q?investigate=20mode=20in=20the=20llm-judge=20README.=20Review=20?= =?UTF-8?q?claim:=20engine/hooks/llm-judge/README.md=20gains=20an=20Invest?= =?UTF-8?q?igate=20mode=20section=20that=20matches=20judge.py.=20Review=20?= =?UTF-8?q?lane:=20docs=20Safety=20invariant:=20Only=20engine/hooks/llm-ju?= =?UTF-8?q?dge/README.md=20changes;=20no=20code=20or=20test=20file=20is=20?= =?UTF-8?q?edited.=20Effectiveness=20measurement:=20grep=20finds=20the=20s?= =?UTF-8?q?ection=20heading=20and=20each=20named=20flag=20in=20the=20READM?= =?UTF-8?q?E.=20Slice=20rationale:=20The=20README=20section=20for=20the=20?= =?UTF-8?q?behavior=20in=20this=20same=20pull=20request,=20which=20catstac?= =?UTF-8?q?k's=20ship-a-detector=20playbook=20requires=20to=20ship=20toget?= =?UTF-8?q?her.=20Architectural=20effect:=20None;=20README=20only.=20Goal:?= =?UTF-8?q?=20Let=20the=20next=20hook=20author=20find=20the=20investigate?= =?UTF-8?q?=20mode=20and=20its=20limits.=20Motivation:=20catstack=20keeps?= =?UTF-8?q?=20every=20hook's=20contract=20in=20its=20README.=20Alternative?= =?UTF-8?q?=20considerations:=20A=20later=20README-only=20pull=20request?= =?UTF-8?q?=20was=20rejected;=20catstack's=20playbook=20counts=20a=20READM?= =?UTF-8?q?E=20that=20lands=20later=20as=20a=20known=20repair=20pattern.?= =?UTF-8?q?=20Implementation=20details:=20Add=20one=20"##=20Investigate=20?= =?UTF-8?q?mode"=20section=20to=20engine/hooks/llm-judge/README.md=20after?= =?UTF-8?q?=20"##=20Runner=20order".=20Non-goals:=20No=20code=20or=20test?= =?UTF-8?q?=20edits.=20Layer:=20docs=20Feature=20state:=20active=20Files:?= =?UTF-8?q?=20-=20engine/hooks/llm-judge/README.md=20Change=20types:=20-?= =?UTF-8?q?=20engine/hooks/llm-judge/README.md:=20modify=20Acceptance=20cr?= =?UTF-8?q?iteria:=20-=20`grep=20-n=20"##=20Investigate=20mode"=20engine/h?= =?UTF-8?q?ooks/llm-judge/README.md`=20exits=200.=20-=20The=20section=20na?= =?UTF-8?q?mes=20--sandbox=20read-only,=20--allowedTools=20Read=20Grep=20G?= =?UTF-8?q?lob,=20--disallowedTools=20Write=20Edit=20NotebookEdit=20Bash,?= =?UTF-8?q?=20and=20600.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context: wf-1789279240280-10/implement-judge-investigate-mode (f2435ab): Let an llm-judge job opt into a read-only investigate mode with its own time limit and working folder. Review claim: A job carrying "mode": "investigate" runs on a second runner set whose Claude argv grants only the Read, Grep and Glob tools and forbids Write, Edit, NotebookEdit and Bash, with the job's own timeout_seconds and cwd; a job lacking that key runs exactly as today. Review lane: behavior Safety invariant: An investigate job never receives a tool that can write, edit, or execute shell text. The Claude runner argv always carries --allowedTools Read Grep Glob and --disallowedTools Write Edit NotebookEdit Bash, the Codex runner always carries --sandbox read-only, and cursor-agent is never used for investigate jobs because it has no read-only switch. Effectiveness measurement: New unit tests pin the exact argv of each investigate runner and prove a default job still gets 60 seconds in an empty temp folder. The PR body pastes the live probe in which claude -p with this argv was asked to Write a file and to touch one through Bash, answered {"wrote":false,"read":true,"bash":false}, and neither target file existed afterwards. Slice rationale: One behavior slice inside the judge, dormant until a hook asks for an investigate job, so a reviewer can judge the permission argv on its own. Architectural effect: llm-judge gains a second, read-only runner set chosen per job; the default runner set and every current caller stay unchanged. Goal: Give later hooks a safe way to ask a model to check a claim by reading files on this machine. Motivation: A backtest of 44 unverified tags from the last 14 days found 33 whose stated blocker was false; most could have been checked by reading a log, a transcript, or a source file already on disk. Today each judge runner gets 60 seconds in an empty temp folder with no tool grants, so it cannot read anything. Alternative considerations: claude --permission-mode plan was rejected after a live probe wrote a plan file under ~/.claude/plans. A Bash prefix allowlist such as Bash(git log:*) was rejected after a live probe blocked even plain git log, so shell access stays out of this slice. Implementation details: In engine/hooks/llm-judge/judge.py add INVESTIGATE_RUNNERS beside DEFAULT_RUNNERS holding the codex and claude argv given under Runner argv. Make runners() take the job mode and pick INVESTIGATE_RUNNERS for "investigate"; CATSTACK_LLM_JUDGE_RUNNERS still replaces whichever set is picked. Make run_runner accept timeout_seconds (default TIMEOUT_SECONDS, capped at INVESTIGATE_TIMEOUT_CAP = 600) and cwd (used only when it is an absolute path to a folder that exists, else the temp folder plus a judge.log line). Thread mode, timeout_seconds and cwd from the job through ask() and run_job(). Runner argv: codex exec --skip-git-repo-check --sandbox read-only -c notify=[] PROMPT, and claude -p --model haiku --settings '{"disableAllHooks": true}' --allowedTools Read Grep Glob --disallowedTools Write Edit NotebookEdit Bash -- PROMPT. Non-goals: No hook asks for an investigate job in this slice. No inbox edit, no change to DEFAULT_RUNNERS, wrong-check-reflect or diu-stop. No shell or network access for the investigator. Layer: domain Feature state: dormant Files: - engine/hooks/llm-judge/judge.py - engine/hooks/llm-judge/tests/test_judge.py Change types: - engine/hooks/llm-judge/judge.py: modify - engine/hooks/llm-judge/tests/test_judge.py: modify Acceptance criteria: - Tests whose names contain "investigate" pass and pin both investigate runner argv exactly. - A test proves a job lacking a mode key still runs DEFAULT_RUNNERS with a 60 second limit. - A test proves timeout_seconds above 600 is capped at 600. - A test whose name contains "missing" proves a cwd that does not exist falls back to a temp folder and logs it. - python3 scripts/check_no_new_comments.py --base origin/main exits 0. Solution: Describe the llm-judge investigate mode in the llm-judge README. Review claim: engine/hooks/llm-judge/README.md gains an Investigate mode section that matches judge.py. Review lane: docs Safety invariant: Only engine/hooks/llm-judge/README.md changes; no code or test file is edited. Effectiveness measurement: grep finds the section heading and each named flag in the README. Slice rationale: The README section for the behavior in this same pull request, which catstack's ship-a-detector playbook requires to ship together. Architectural effect: None; README only. Goal: Let the next hook author find the investigate mode and its limits. Motivation: catstack keeps every hook's contract in its README. Alternative considerations: A later README-only pull request was rejected; catstack's playbook counts a README that lands later as a known repair pattern. Implementation details: Add one "## Investigate mode" section to engine/hooks/llm-judge/README.md after "## Runner order". Non-goals: No code or test edits. Layer: docs Feature state: active Files: - engine/hooks/llm-judge/README.md Change types: - engine/hooks/llm-judge/README.md: modify Acceptance criteria: - `grep -n "## Investigate mode" engine/hooks/llm-judge/README.md` exits 0. - The section names --sandbox read-only, --allowedTools Read Grep Glob, --disallowedTools Write Edit NotebookEdit Bash, and 600. Invoker-Finalize-Id: c7674a0c-7e5f-4540-b149-295bbac2133e --- engine/hooks/llm-judge/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/engine/hooks/llm-judge/README.md b/engine/hooks/llm-judge/README.md index fe5ee99b..31eba814 100644 --- a/engine/hooks/llm-judge/README.md +++ b/engine/hooks/llm-judge/README.md @@ -94,6 +94,25 @@ most 300 characters, taken from the end of stderr or the error text. Tests use it to plug in small fake runners. If it is set but not that shape, `ask` raises `ValueError` instead of quietly falling back to the real runners. +## Investigate mode + +A job opts in with `"mode": "investigate"`. It uses a read-only runner set: + +1. **codex**: `codex exec --skip-git-repo-check --sandbox read-only -c notify=[] PROMPT` +2. **claude**: `claude -p --model haiku --settings '{"disableAllHooks": true}' --allowedTools Read Grep Glob --disallowedTools Write Edit NotebookEdit Bash -- PROMPT` + +`cursor-agent` is not used because it has no read-only switch. + +An investigate job may carry `timeout_seconds`. The judge caps it at 600 +seconds. If it is missing or not a number, the runner gets 60 seconds. + +An investigate job may carry `cwd`. The judge uses it only when it is an +absolute path to a folder that exists. Otherwise the runner uses a fresh temp +folder, and `judge.log` gets a line saying the cwd was refused. + +`CATSTACK_LLM_JUDGE_RUNNERS` still replaces the selected runner set. A job that +gets no answer from any runner still comes back `unchecked`. + ## Three outcomes `verdict(job, result)` turns an `ask` result into one of: From 68035e29614ec95dd9816d1045a45b0a87e31e99 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:05:38 +0000 Subject: [PATCH 4/6] =?UTF-8?q?invoker:=20wf-1789279240280-10/verify-judge?= =?UTF-8?q?-full-suite=20=E2=80=94=20Proof=20step=202=20for=20the=20llm-ju?= =?UTF-8?q?dge=20investigate=20mode.=20Review=20claim:=20Every=20llm-judge?= =?UTF-8?q?=20test=20still=20passes=20after=20the=20change.=20Review=20lan?= =?UTF-8?q?e:=20proof=20Safety=20invariant:=20Proof-only;=20adds=20no=20pr?= =?UTF-8?q?oduct=20behavior.=20Effectiveness=20measurement:=20The=20whole?= =?UTF-8?q?=20llm-judge=20test=20folder=20exits=200.=20Slice=20rationale:?= =?UTF-8?q?=20One=20proof=20step=20guarding=20current=20judge=20callers.?= =?UTF-8?q?=20Architectural=20effect:=20None;=20verification=20only.=20Goa?= =?UTF-8?q?l:=20Prove=20default=20jobs=20behave=20as=20before.=20Motivatio?= =?UTF-8?q?n:=20wrong-check-reflect=20and=20diu-stop=20already=20depend=20?= =?UTF-8?q?on=20the=20judge.=20Alternative=20considerations:=20Running=20o?= =?UTF-8?q?nly=20the=20new=20tests=20was=20rejected;=20it=20would=20miss?= =?UTF-8?q?=20a=20regression=20in=20the=20default=20runner=20set.=20Implem?= =?UTF-8?q?entation=20details:=20Run=20the=20whole=20llm-judge=20test=20fo?= =?UTF-8?q?lder.=20Non-goals:=20No=20product=20edits=20here;=20proof=20onl?= =?UTF-8?q?y.=20Layer:=20app=5Fregression=20Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: c94db032-2d49-4594-b6e7-b3ac43e7b56b From 258b69e0e624433f98541b73039937e86d5c7599 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:06:18 +0000 Subject: [PATCH 5/6] =?UTF-8?q?invoker:=20wf-1789279240280-10/verify-judge?= =?UTF-8?q?-no-new-comments=20=E2=80=94=20Proof=20step=203=20for=20the=20l?= =?UTF-8?q?lm-judge=20investigate=20mode.=20Review=20claim:=20The=20change?= =?UTF-8?q?=20adds=20no=20explanatory=20code=20comments.=20Review=20lane:?= =?UTF-8?q?=20proof=20Safety=20invariant:=20Proof-only;=20adds=20no=20prod?= =?UTF-8?q?uct=20behavior.=20Effectiveness=20measurement:=20The=20comment?= =?UTF-8?q?=20gate=20exits=200=20against=20origin/main.=20Slice=20rational?= =?UTF-8?q?e:=20One=20proof=20step=20for=20catstack's=20required=20comment?= =?UTF-8?q?=20gate.=20Architectural=20effect:=20None;=20verification=20onl?= =?UTF-8?q?y.=20Goal:=20Keep=20the=20CI=20comment=20gate=20green.=20Motiva?= =?UTF-8?q?tion:=20CI=20runs=20this=20gate=20on=20every=20pull=20request.?= =?UTF-8?q?=20Alternative=20considerations:=20Waiting=20for=20CI=20was=20r?= =?UTF-8?q?ejected;=20it=20is=20slower=20to=20find.=20Implementation=20det?= =?UTF-8?q?ails:=20Run=20the=20comment=20gate=20against=20origin/main.=20N?= =?UTF-8?q?on-goals:=20No=20product=20edits=20here;=20proof=20only.=20Laye?= =?UTF-8?q?r:=20app=5Fregression=20Feature=20state:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 492bf3e7-0cff-46b3-b0d1-89687e47b6b7 From 94a1361c12138f32edd5867a664feb2726982f2c Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 13 Sep 2026 06:06:59 +0000 Subject: [PATCH 6/6] =?UTF-8?q?invoker:=20wf-1789279240280-10/scrub-handof?= =?UTF-8?q?f-artifacts=20=E2=80=94=20Check=20that=20no=20ephemeral=20inter?= =?UTF-8?q?-task=20handoff=20files=20remain=20before=20the=20merge=20gate.?= =?UTF-8?q?=20Review=20claim:=20Only=20ephemeral=20handoff=20files=20are?= =?UTF-8?q?=20checked;=20no=20product=20or=20test=20file=20is=20touched.?= =?UTF-8?q?=20Review=20lane:=20cleanup=20Safety=20invariant:=20The=20scrub?= =?UTF-8?q?=20script=20runs=20read-only=20and=20never=20touches=20the=20ho?= =?UTF-8?q?me=20Invoker=20ledger.=20Effectiveness=20measurement:=20The=20s?= =?UTF-8?q?crub=20script=20exits=200=20and=20reports=20no=20handoff=20path?= =?UTF-8?q?.=20Slice=20rationale:=20One=20cleanup=20slice=20required=20on?= =?UTF-8?q?=20every=20implementation=20plan=20that=20opens=20a=20pull=20re?= =?UTF-8?q?quest,=20kept=20separate=20from=20behavior=20and=20proof=20work?= =?UTF-8?q?.=20Architectural=20effect:=20None.=20Goal:=20Leave=20a=20clean?= =?UTF-8?q?=20worktree=20for=20the=20pull=20request.=20Motivation:=20Ephem?= =?UTF-8?q?eral=20inter-task=20files=20must=20not=20leak=20into=20the=20re?= =?UTF-8?q?viewed=20diff.=20Alternative=20considerations:=20Leaving=20this?= =?UTF-8?q?=20step=20out=20was=20rejected;=20it=20is=20a=20hard=20requirem?= =?UTF-8?q?ent=20for=20every=20plan=20that=20opens=20a=20pull=20request.?= =?UTF-8?q?=20Implementation=20details:=20Run=20the=20repository's=20hando?= =?UTF-8?q?ff-scrub=20script=20without=20--apply.=20Non-goals:=20No=20prod?= =?UTF-8?q?uct=20edits=20in=20this=20task.=20Layer:=20docs=20Feature=20sta?= =?UTF-8?q?te:=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 2aafd01f-794a-46f1-8763-0a04ed358458