diff --git a/docs/ecosystem.md b/docs/ecosystem.md index 32b391fa..13071260 100644 --- a/docs/ecosystem.md +++ b/docs/ecosystem.md @@ -92,6 +92,7 @@ again. | `hook-freshness` | hook (advisory) | | `hook-health` | hook (advisory) | | `llm-judge` | hook (shared background model judge; its inbox delivers finished verdicts on the next turn: Claude `UserPromptSubmit`, Cursor `stop`, Codex `notify`) | +| `unverified-tag-check` | hook (advisory; background read-only check of each unverified tag, reported through llm-judge's inbox) | | `engine/CLAUDE.core.md` | global hand-written Claude rules | | `scripts/`, `always-on/`, `cursor/rules/` (repo root), root `install.sh` | runtime (engine-owned entrypoints at root for CI) | diff --git a/engine/hooks/unverified-tag-check/README.md b/engine/hooks/unverified-tag-check/README.md new file mode 100644 index 00000000..2b40da65 --- /dev/null +++ b/engine/hooks/unverified-tag-check/README.md @@ -0,0 +1,54 @@ +# unverified-tag-check + +Check every well-formed `CAT-UNVERIFIED` tag in the background and report +whether the blocker held. + +The hook fires on a finished assistant reply from Claude `Stop`, Cursor +`stop`, and Codex `notify` (`agent-turn-complete`). It looks for a tag shaped +like: + +```text +{{CAT-UNVERIFIED: -- cannot verify: }} +``` + +Malformed tags stay silent. Tags inside closed code fences or inline code stay +silent. A claim already checked in the same transcript within 2 hours stays +silent. Anything after the first 3 tags in one reply is ignored. + +The live reply is never blocked or delayed. The hook hands one read-only +investigation job to [`llm-judge`](../llm-judge/README.md) for each new tag and +returns. + +On the next step, the shared `llm-judge` inbox shows the job's `on_hit` text +and the checker's report sentence: + +```text +unverified-tag-check: checked "". Tell the user this result in plain words: +``` + +The checker reports whether the stated blocker was real and whether the claim +is true, false, or unknown from readable evidence. It only gets Read, Grep, and +Glob, so it cannot run shell or network checks. If local files and transcripts +cannot answer the question, it says unknown. + +Fail-open. An unreadable reply, missing transcript, broken judge, broken state, +or missing runner hands off nothing or comes back unchecked. It never becomes a +true result, and it never blocks the reply. + +## Files + +- `detect.py` - tag extraction, two-hour state, and `llm-judge` job enqueue +- `claude_stop_check.py` - Claude `Stop` +- `cursor_session.py` - Cursor `stop` / `sessionEnd` +- `codex_notify.py` - Codex `notify` +- `install_claude_hook.py` / `install_cursor_hook.py` / `install_codex_notify.py` + +## Install + +`./install.sh` from the repo root. Restart the harness. + +## Tests + +```sh +python3 -m unittest discover -s engine/hooks/unverified-tag-check/tests -v +``` diff --git a/engine/hooks/unverified-tag-check/claude.hook.json b/engine/hooks/unverified-tag-check/claude.hook.json new file mode 100644 index 00000000..e413d839 --- /dev/null +++ b/engine/hooks/unverified-tag-check/claude.hook.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "Stop": [ + { + "matcher": "*", + "hooks": [ + { + "type": "command", + "command": "python3 $HOME/.claude/hooks/unverified-tag-check/claude_stop_check.py", + "timeout": 10 + } + ] + } + ] + } +} diff --git a/engine/hooks/unverified-tag-check/claude_stop_check.py b/engine/hooks/unverified-tag-check/claude_stop_check.py new file mode 100644 index 00000000..ba000e02 --- /dev/null +++ b/engine/hooks/unverified-tag-check/claude_stop_check.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import sys + +from detect import try_check_reply + + +def main() -> None: + try: + payload = json.load(sys.stdin) + except Exception as exc: + print(f"catstack-hook-error unverified-tag-check: {type(exc).__name__}: {exc}", file=sys.stderr) + return + payload = payload if isinstance(payload, dict) else {} + try_check_reply(payload) + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/unverified-tag-check/codex_notify.py b/engine/hooks/unverified-tag-check/codex_notify.py new file mode 100644 index 00000000..78c88c25 --- /dev/null +++ b/engine/hooks/unverified-tag-check/codex_notify.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import subprocess +import sys + +from detect import try_check_reply + + +def main() -> None: + if len(sys.argv) < 2: + return + raw = sys.argv[-1] + chain = sys.argv[1:-1] + + if chain: + try: + subprocess.run(chain + [raw], timeout=5, check=False) + except Exception as exc: + print(f"catstack-hook-error unverified-tag-check: {type(exc).__name__}: {exc}", file=sys.stderr) + + try: + payload = json.loads(raw) + except Exception as exc: + print(f"catstack-hook-error unverified-tag-check: {type(exc).__name__}: {exc}", file=sys.stderr) + return + + if payload.get("type") != "agent-turn-complete": + return + + try_check_reply(payload) + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/unverified-tag-check/cursor_session.py b/engine/hooks/unverified-tag-check/cursor_session.py new file mode 100644 index 00000000..eca49d16 --- /dev/null +++ b/engine/hooks/unverified-tag-check/cursor_session.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import sys + +from detect import try_check_reply + + +def main() -> None: + try: + payload = json.load(sys.stdin) + except Exception as exc: + print(f"catstack-hook-error unverified-tag-check: {type(exc).__name__}: {exc}", file=sys.stderr) + print(json.dumps({"followup_message": ""})) + return + payload = payload if isinstance(payload, dict) else {} + try_check_reply(payload) + print(json.dumps({"followup_message": ""})) + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/unverified-tag-check/detect.py b/engine/hooks/unverified-tag-check/detect.py new file mode 100644 index 00000000..826e8084 --- /dev/null +++ b/engine/hooks/unverified-tag-check/detect.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import functools +import hashlib +import importlib.util +import json +import os +import re +import sys +import time +import uuid + +HOOK_NAME = "unverified-tag-check" +HOOKS_DIR = os.path.dirname(os.path.abspath(__file__)) +LLM_JUDGE_PATH = os.path.join(os.path.dirname(HOOKS_DIR), "llm-judge", "judge.py") +MARKERS_DIR = os.path.join(os.path.dirname(HOOKS_DIR), "_markers") +STATE_ENV = "CATSTACK_UNVERIFIED_TAG_CHECK_STATE_DIR" +STATE_TTL_SECONDS = 7200 +PROMPT_LIMIT = 8000 + +sys.path.insert(0, MARKERS_DIR) + +import markers + +REASON_SPLIT_RE = re.compile(r"cannot\s+verify\s*:", re.IGNORECASE) +TAG_BODY_RE = re.compile(r"^\{\{\s*CAT-UNVERIFIED\b(?P[^}]*)\}\}$", re.IGNORECASE | re.DOTALL) +INLINE_CODE_RE = re.compile(r"`[^`\n]*`") + + +def _stderr(message: str) -> None: + print(message, file=sys.stderr) + + +def _blank_span(chars: list[str], start: int, end: int) -> None: + for index in range(start, end): + if chars[index] != "\n": + chars[index] = " " + + +def strip_code(text: str) -> str: + chars = list(text) + index = 0 + while True: + start = text.find("```", index) + if start < 0: + break + end = text.find("```", start + 3) + if end < 0: + break + _blank_span(chars, start, end + 3) + index = end + 3 + fenced = "".join(chars) + for match in INLINE_CODE_RE.finditer(fenced): + _blank_span(chars, match.start(), match.end()) + return "".join(chars) + + +def _message_text(data: dict) -> str: + message = data.get("message") + content = message.get("content") if isinstance(message, dict) else data.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(block.get("text") or "") + elif isinstance(block, str): + parts.append(block) + return "\n".join(parts) + return "" + + +def _is_assistant_line(data: dict) -> bool: + if data.get("type") == "assistant": + return True + message = data.get("message") + return isinstance(message, dict) and message.get("role") == "assistant" + + +def _last_assistant_from_transcript(path: str) -> str: + last = "" + try: + with open(path, encoding="utf-8") as handle: + for line in handle: + try: + data = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(data, dict) or not _is_assistant_line(data): + continue + text = _message_text(data) + if text.strip(): + last = text + except OSError: + return "" + return last + + +def resolve_transcript(payload: dict) -> str: + agent = payload.get("agent_transcript_path") + if isinstance(agent, str): + return agent if os.path.isfile(agent) else "" + direct = payload.get("transcript_path") or payload.get("transcriptPath") + if isinstance(direct, str) and os.path.isfile(direct): + return direct + return "" + + +def reply_text(payload: dict) -> str: + value = payload.get("last_assistant_message") + if isinstance(value, str) and value.strip(): + return value + return _last_assistant_from_transcript(str(payload.get("transcript_path") or "")) + + +def _tag_body(tag: str) -> str: + match = TAG_BODY_RE.match(tag) + if not match: + return "" + body = match.group("body").strip() + if body.startswith(":"): + body = body[1:] + return body.strip() + + +def tags(text: str) -> list[dict]: + out = [] + for tag in markers.well_formed_tags(strip_code(text)): + body = _tag_body(tag) + parts = REASON_SPLIT_RE.split(body, 1) + if len(parts) != 2: + continue + claim = parts[0].strip(" -:\t\r\n") + blocker = parts[1].strip(" -:\t\r\n") + out.append({"claim": claim, "blocker": blocker, "tag": tag}) + if len(out) == 3: + break + return out + + +def normalize_claim(claim: str) -> str: + return re.sub(r"\W+", " ", claim.lower()).strip() + + +def state_root() -> str: + return os.environ.get(STATE_ENV) or os.path.join(os.path.expanduser("~"), ".cache", "catstack-unverified-tag-check") + + +def state_path(transcript: str) -> str: + digest = hashlib.sha1(transcript.encode("utf-8")).hexdigest()[:16] + return os.path.join(state_root(), f"{digest}.json") + + +def read_state(transcript: str, now: float | None = None) -> dict[str, float]: + now = time.time() if now is None else now + path = state_path(transcript) + try: + with open(path, encoding="utf-8") as handle: + loaded = json.load(handle) + except (OSError, ValueError): + return {} + if not isinstance(loaded, dict): + return {} + out: dict[str, float] = {} + for claim, seen_at in loaded.items(): + if not isinstance(claim, str) or isinstance(seen_at, bool) or not isinstance(seen_at, (int, float)): + return {} + if seen_at > now: + continue + if now - float(seen_at) <= STATE_TTL_SECONDS: + out[claim] = float(seen_at) + return out + + +def write_state(transcript: str, state: dict[str, float]) -> None: + path = state_path(transcript) + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(state, handle, sort_keys=True) + except Exception as exc: + print(f"unverified-tag-check: state not written: {type(exc).__name__}: {exc}", file=sys.stderr) + + +def _clip(text: str, limit: int) -> str: + text = " ".join(str(text or "").split()) + if len(text) <= limit: + return text + return text[: max(0, limit - 3)].rstrip() + "..." + + +def _paragraph_holding_tag(text: str, tag: str) -> str: + paragraphs = re.split(r"\n\s*\n", text) + for paragraph in paragraphs: + if tag in strip_code(paragraph): + return paragraph + for paragraph in paragraphs: + if "CAT-UNVERIFIED" in paragraph and REASON_SPLIT_RE.search(paragraph): + return paragraph + return text + + +def _prompt(payload: dict, tag: dict) -> str: + text = reply_text(payload) + transcript = resolve_transcript(payload) + cwd = payload.get("cwd") or "" + paragraph = _clip(_paragraph_holding_tag(text, str(tag.get("tag") or "")), 2000) + claim = _clip(str(tag.get("claim") or ""), 1500) + blocker = _clip(str(tag.get("blocker") or ""), 1500) + prompt = ( + "You check one excuse an AI agent gave for not verifying a claim. You may only read. " + f"Claim: {claim}. Stated blocker: {blocker}. The reply paragraph: {paragraph}. " + f"The agent's transcript (JSONL) is at {transcript}; its working folder was {cwd}. " + "First decide whether the blocker was real: look for a file, log, transcript line, or earlier tool result on this machine that the agent could have used to check the claim, or that shows the 'impossible' thing already worked. " + "Then decide whether the claim itself is true, false, or unknown from what you can read. " + "Quote evidence as file:line or a short exact quote; with no evidence, say unknown. " + "Answer with exactly one JSON object on the last line:\n" + "{\"blocker_false\": true|false, \"claim_status\": \"true\"|\"false\"|\"unknown\", \"report\": \"\"}" + ) + return prompt[:PROMPT_LIMIT] + + +def build_job(payload: dict, tag: dict) -> dict: + claim = str(tag.get("claim") or "") + return { + "id": uuid.uuid4().hex, + "hook": HOOK_NAME, + "transcript": resolve_transcript(payload), + "mode": "investigate", + "timeout_seconds": 300, + "cwd": payload.get("cwd") or "", + "hit_if_all_true": [], + "on_hit": f"unverified-tag-check: checked \"{_clip(claim, 120)}\". Tell the user this result in plain words:", + "prompt": _prompt(payload, tag), + } + + +@functools.cache +def _judge(): + spec = importlib.util.spec_from_file_location("llm_judge", LLM_JUDGE_PATH) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load llm-judge from {LLM_JUDGE_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def check_reply(payload: dict) -> list[str]: + if not isinstance(payload, dict): + return [] + transcript = resolve_transcript(payload) + if not transcript: + _stderr("unverified-tag-check: no transcript, reply not checked") + return [] + text = reply_text(payload) + if not text.strip(): + return [] + now = time.time() + state = read_state(transcript, now=now) + seen = set(state) + ids = [] + for tag in tags(text): + normalized = normalize_claim(str(tag.get("claim") or "")) + if not normalized or normalized in seen: + continue + job_id = _judge().enqueue(build_job(payload, tag)) + if job_id is not None: + ids.append(job_id) + state[normalized] = now + seen.add(normalized) + if ids: + write_state(transcript, state) + return ids + + +def try_check_reply(payload: dict) -> None: + try: + check_reply(payload) + except Exception as exc: + print(f"catstack-hook-error {HOOK_NAME}: {type(exc).__name__}: {exc}", file=sys.stderr) diff --git a/engine/hooks/unverified-tag-check/install_claude_hook.py b/engine/hooks/unverified-tag-check/install_claude_hook.py new file mode 100644 index 00000000..44ef82bb --- /dev/null +++ b/engine/hooks/unverified-tag-check/install_claude_hook.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +SETTINGS_PATH = os.path.expanduser("~/.claude/settings.json") +FRAGMENT_PATH = os.path.join(HERE, "claude.hook.json") +MARKER = "unverified-tag-check/claude_stop_check.py" + + +def _is_ours(entry: dict) -> bool: + return any(MARKER in h.get("command", "") for h in entry.get("hooks", [])) + + +def merge_hook(settings: dict, fragment: dict) -> bool: + entry_list = settings.setdefault("hooks", {}).setdefault("Stop", []) + new_entries = fragment.get("hooks", {}).get("Stop", []) + before = json.dumps(entry_list, sort_keys=True) + kept = [e for e in entry_list if not _is_ours(e)] + entry_list[:] = kept + new_entries + return json.dumps(entry_list, sort_keys=True) != before + + +def main() -> None: + settings: dict = {} + if os.path.exists(SETTINGS_PATH): + with open(SETTINGS_PATH) as handle: + settings = json.load(handle) + with open(FRAGMENT_PATH) as handle: + fragment = json.load(handle) + if not merge_hook(settings, fragment): + print("ok claude Stop unverified-tag-check already up to date") + return + os.makedirs(os.path.dirname(SETTINGS_PATH), exist_ok=True) + with open(SETTINGS_PATH, "w") as handle: + json.dump(settings, handle, indent=2) + handle.write("\n") + print("link claude Stop unverified-tag-check merged into settings.json") + print(" (restart Claude Code to pick up the change)") + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/unverified-tag-check/install_codex_notify.py b/engine/hooks/unverified-tag-check/install_codex_notify.py new file mode 100644 index 00000000..abb2b546 --- /dev/null +++ b/engine/hooks/unverified-tag-check/install_codex_notify.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +import re + +MARKER = "unverified-tag-check/codex_notify.py" +CONFIG_PATH = os.path.expanduser("~/.codex/config.toml") +SCRIPT_PATH = os.path.expanduser("~/.codex/hooks/unverified-tag-check/codex_notify.py") + +NOTIFY_RE = re.compile(r"^notify\s*=\s*(\[.*\])\s*$", re.MULTILINE) +SECTION_RE = re.compile(r"^\[", re.MULTILINE) + + +def compute_notify_update(config_text: str, script_path: str): + match = NOTIFY_RE.search(config_text) + + if match: + current = json.loads(match.group(1)) + if any(MARKER in str(item) for item in current): + return config_text, False, "codex notify unverified-tag-check already wired, skipping" + new_array = ["python3", script_path] + current + new_line = "notify = " + json.dumps(new_array) + new_text = config_text[: match.start()] + new_line + config_text[match.end() :] + return new_text, True, f"codex notify unverified-tag-check wired (chaining {len(current)} prior arg(s))" + + new_array = ["python3", script_path] + new_line = "notify = " + json.dumps(new_array) + "\n" + section = SECTION_RE.search(config_text) + if section: + new_text = config_text[: section.start()] + new_line + config_text[section.start() :] + else: + sep = "\n" if config_text and not config_text.endswith("\n") else "" + new_text = config_text + sep + new_line + return new_text, True, "codex notify unverified-tag-check added (no prior notify command found)" + + +def main() -> None: + if not os.path.exists(CONFIG_PATH): + print(f"skip {CONFIG_PATH} does not exist, nothing to wire") + return + + with open(CONFIG_PATH) as handle: + text = handle.read() + + new_text, changed, message = compute_notify_update(text, SCRIPT_PATH) + prefix = "link " if changed else "ok " + print(prefix + message) + + if changed: + with open(CONFIG_PATH, "w") as handle: + handle.write(new_text) + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/unverified-tag-check/install_cursor_hook.py b/engine/hooks/unverified-tag-check/install_cursor_hook.py new file mode 100644 index 00000000..8180e3de --- /dev/null +++ b/engine/hooks/unverified-tag-check/install_cursor_hook.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os + +HOOKS_PATH = os.path.expanduser("~/.cursor/hooks.json") +MARKER = "unverified-tag-check/cursor_session.py" +COMMAND = "python3 $HOME/.cursor/hooks/unverified-tag-check/cursor_session.py" + +STOP_ENTRY = { + "command": COMMAND, + "timeout": 10, + "loop_limit": 1, +} +SESSION_END_ENTRY = { + "command": COMMAND + " sessionEnd", + "timeout": 10, +} + + +def _is_ours(entry: dict) -> bool: + return MARKER in str(entry.get("command", "")) + + +def merge_list(existing: list, incoming: dict) -> list: + kept = [e for e in existing if not _is_ours(e)] + return kept + [incoming] + + +def main() -> None: + if os.path.islink(HOOKS_PATH): + print( + "skip cursor hooks.json is a symlink; bug-complaint-leak installer materializes it first" + ) + return + data: dict = {"version": 1, "hooks": {}} + if os.path.exists(HOOKS_PATH): + with open(HOOKS_PATH) as handle: + loaded = json.load(handle) + if isinstance(loaded, dict): + data = loaded + data.setdefault("version", 1) + hooks = data.setdefault("hooks", {}) + changed = False + for key, incoming in (("stop", STOP_ENTRY), ("sessionEnd", SESSION_END_ENTRY)): + before = json.dumps(hooks.get(key, []), sort_keys=True) + hooks[key] = merge_list(list(hooks.get(key, [])), incoming) + after = json.dumps(hooks[key], sort_keys=True) + if before != after: + changed = True + print(f"link cursor {key} unverified-tag-check merged") + else: + print(f"ok cursor {key} unverified-tag-check already up to date") + if not changed: + return + os.makedirs(os.path.dirname(HOOKS_PATH), exist_ok=True) + with open(HOOKS_PATH, "w") as handle: + json.dump(data, handle, indent=2) + handle.write("\n") + print(" (restart Cursor to pick up the change)") + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/unverified-tag-check/tests/test_hooks.py b/engine/hooks/unverified-tag-check/tests/test_hooks.py new file mode 100644 index 00000000..12118c10 --- /dev/null +++ b/engine/hooks/unverified-tag-check/tests/test_hooks.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import contextlib +import importlib +import io +import json +import os +import stat +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +HOOKS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, HOOKS_DIR) + +claude_stop_check = importlib.import_module("claude_stop_check") +codex_notify = importlib.import_module("codex_notify") +cursor_session = importlib.import_module("cursor_session") +detect = importlib.import_module("detect") + +POSITIVE_ONE = "{{CAT-UNVERIFIED: DO1's repair worker is off or stuck -- cannot verify: can't log in to DO1 to look}}" +POSITIVE_TWO = "{{CAT-UNVERIFIED: the fixed `pnpm` command ran instead of the repo's own setup command -- cannot verify: I did not read Invoker's code}}" + + +def transcript_line(role: str, text: str) -> str: + return json.dumps({"type": role, "message": {"role": role, "content": [{"type": "text", "text": text}]}}) + + +class TestUnverifiedTagCheck(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.judge_state = os.path.join(self.tmp.name, "judge") + self.detector_state = os.path.join(self.tmp.name, "state") + self.env = patch.dict(os.environ, { + "CATSTACK_LLM_JUDGE_STATE_DIR": self.judge_state, + "CATSTACK_UNVERIFIED_TAG_CHECK_STATE_DIR": self.detector_state, + }) + self.env.start() + self.enqueued = [] + detect._judge.cache_clear() + self.judge = detect._judge() + self.enqueue_patch = patch.object(self.judge, "enqueue", side_effect=self.enqueue) + self.enqueue_patch.start() + + def tearDown(self): + self.enqueue_patch.stop() + detect._judge.cache_clear() + self.env.stop() + self.tmp.cleanup() + + def enqueue(self, job: dict) -> str: + self.enqueued.append(dict(job)) + return job["id"] + + def write_transcript(self, text: str = "", name: str = "session.jsonl") -> str: + path = os.path.join(self.tmp.name, name) + with open(path, "w", encoding="utf-8") as handle: + if text: + handle.write(transcript_line("assistant", text) + "\n") + return path + + def check(self, text: str, path: str | None = None) -> list[str]: + transcript = path or self.write_transcript() + return detect.check_reply({ + "transcript_path": transcript, + "last_assistant_message": text, + "cwd": self.tmp.name, + }) + + def test_hit_one_job_per_tag(self): + path = self.write_transcript() + ids = self.check(f"{POSITIVE_ONE}\n\n{POSITIVE_TWO}", path) + self.assertEqual(ids, [job["id"] for job in self.enqueued]) + self.assertEqual(len(self.enqueued), 2) + self.assertEqual(set(self.enqueued[0]), { + "id", + "hook", + "transcript", + "mode", + "timeout_seconds", + "cwd", + "hit_if_all_true", + "on_hit", + "prompt", + }) + self.assertEqual(self.enqueued[0]["hook"], "unverified-tag-check") + self.assertEqual(self.enqueued[0]["transcript"], path) + self.assertIn("DO1's repair worker is off or stuck", self.enqueued[0]["on_hit"]) + + def test_hit_job_uses_investigate_mode(self): + self.check(POSITIVE_ONE) + job = self.enqueued[0] + self.assertEqual(job["mode"], "investigate") + self.assertEqual(job["timeout_seconds"], 300) + self.assertEqual(job["hit_if_all_true"], []) + self.assertIn('"blocker_false": true|false', job["prompt"]) + self.assertIn('"claim_status": "true"|"false"|"unknown"', job["prompt"]) + self.assertIn('"report"', job["prompt"]) + + def test_hit_reads_last_assistant_text_from_transcript(self): + path = self.write_transcript(POSITIVE_ONE) + ids = detect.check_reply({"transcript_path": path, "cwd": self.tmp.name}) + self.assertEqual(len(ids), 1) + self.assertEqual(len(self.enqueued), 1) + + def test_hit_expired_state_entry_gets_checked_again(self): + path = self.write_transcript() + os.makedirs(self.detector_state, exist_ok=True) + with open(detect.state_path(path), "w", encoding="utf-8") as handle: + json.dump({"do1 s repair worker is off or stuck": time.time() - 7201}, handle) + self.check(POSITIVE_ONE, path) + self.assertEqual(len(self.enqueued), 1) + + def test_silent_malformed_tag_with_no_reason(self): + self.check("{{CAT-UNVERIFIED: claim -- cannot verify: }}") + self.assertEqual(self.enqueued, []) + + def test_silent_tag_inside_closed_fence(self): + self.check(f"```text\n{POSITIVE_ONE}\n```") + self.assertEqual(self.enqueued, []) + + def test_silent_tag_inside_inline_code(self): + self.check(f"`{POSITIVE_ONE}`") + self.assertEqual(self.enqueued, []) + + def test_silent_same_claim_twice_in_one_transcript(self): + path = self.write_transcript() + self.check(POSITIVE_ONE, path) + self.enqueued.clear() + self.check(POSITIVE_ONE, path) + self.assertEqual(self.enqueued, []) + + def test_silent_reply_with_no_tag(self): + self.check("I could not check this one.") + self.assertEqual(self.enqueued, []) + + def test_silent_fourth_tag_in_one_reply(self): + text = "\n".join( + f"{{{{CAT-UNVERIFIED: claim {index} -- cannot verify: blocker {index}}}}}" + for index in range(4) + ) + self.check(text) + self.assertEqual(len(self.enqueued), 3) + self.assertNotIn("claim 3", "\n".join(job["on_hit"] for job in self.enqueued)) + + def test_unreadable_state_file_reads_as_empty(self): + path = self.write_transcript() + os.makedirs(self.detector_state, exist_ok=True) + state_path = detect.state_path(path) + with open(state_path, "w", encoding="utf-8") as handle: + json.dump({"do1 s repair worker is off or stuck": time.time()}, handle) + os.chmod(state_path, 0) + try: + self.check(POSITIVE_ONE, path) + finally: + os.chmod(state_path, stat.S_IRUSR | stat.S_IWUSR) + self.assertEqual(len(self.enqueued), 1) + + def test_missing_transcript_hands_off_nothing(self): + err = io.StringIO() + missing = os.path.join(self.tmp.name, "missing.jsonl") + with contextlib.redirect_stderr(err): + ids = detect.check_reply({"transcript_path": missing, "last_assistant_message": POSITIVE_ONE}) + self.assertEqual(ids, []) + self.assertEqual(self.enqueued, []) + self.assertIn("unverified-tag-check: no transcript, reply not checked", err.getvalue()) + + def test_entry_script_exits_zero_on_malformed_stdin(self): + for module in (claude_stop_check, cursor_session): + err = io.StringIO() + out = io.StringIO() + with patch.object(sys, "stdin", io.StringIO("not-json")): + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + module.main() + self.assertIn("catstack-hook-error unverified-tag-check: JSONDecodeError:", err.getvalue()) + err = io.StringIO() + with patch.object(sys, "argv", ["codex_notify.py", "not-json"]): + with contextlib.redirect_stderr(err): + codex_notify.main() + self.assertIn("catstack-hook-error unverified-tag-check: JSONDecodeError:", err.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/install.sh b/install.sh index d5e2d1bb..8f4f3e2a 100755 --- a/install.sh +++ b/install.sh @@ -231,6 +231,7 @@ link_item "pr-schema-gate" "$REPO_DIR/engine/hooks/pr-schema-gate" "$HOME/.claud link_item "history-claim-check" "$REPO_DIR/engine/hooks/history-claim-check" "$HOME/.claude/hooks/history-claim-check" link_item "external-claim-gate" "$REPO_DIR/engine/hooks/external-claim-gate" "$HOME/.claude/hooks/external-claim-gate" link_item "wrong-check-reflect" "$REPO_DIR/engine/hooks/wrong-check-reflect" "$HOME/.claude/hooks/wrong-check-reflect" +link_item "unverified-tag-check" "$REPO_DIR/engine/hooks/unverified-tag-check" "$HOME/.claude/hooks/unverified-tag-check" link_item "llm-judge" "$REPO_DIR/engine/hooks/llm-judge" "$HOME/.claude/hooks/llm-judge" link_item "hook-health" "$REPO_DIR/engine/hooks/hook-health" "$HOME/.claude/hooks/hook-health" link_item "build-the-lever" "$REPO_DIR/engine/hooks/build-the-lever" "$HOME/.claude/hooks/build-the-lever" @@ -275,6 +276,7 @@ link_item "scope-lock" "$REPO_DIR/engine/hooks/scope-lock" "$HOME/.cursor/hooks/ link_item "auto-pr" "$REPO_DIR/engine/hooks/auto-pr" "$HOME/.cursor/hooks/auto-pr" link_item "pr-schema-gate" "$REPO_DIR/engine/hooks/pr-schema-gate" "$HOME/.cursor/hooks/pr-schema-gate" link_item "wrong-check-reflect" "$REPO_DIR/engine/hooks/wrong-check-reflect" "$HOME/.cursor/hooks/wrong-check-reflect" +link_item "unverified-tag-check" "$REPO_DIR/engine/hooks/unverified-tag-check" "$HOME/.cursor/hooks/unverified-tag-check" link_item "llm-judge" "$REPO_DIR/engine/hooks/llm-judge" "$HOME/.cursor/hooks/llm-judge" link_item "hook-health" "$REPO_DIR/engine/hooks/hook-health" "$HOME/.cursor/hooks/hook-health" link_item "build-the-lever" "$REPO_DIR/engine/hooks/build-the-lever" "$HOME/.cursor/hooks/build-the-lever" @@ -290,6 +292,7 @@ link_item "scope-lock" "$REPO_DIR/engine/hooks/scope-lock" "$HOME/.codex/hooks/s link_item "auto-pr" "$REPO_DIR/engine/hooks/auto-pr" "$HOME/.codex/hooks/auto-pr" link_item "pr-schema-gate" "$REPO_DIR/engine/hooks/pr-schema-gate" "$HOME/.codex/hooks/pr-schema-gate" link_item "wrong-check-reflect" "$REPO_DIR/engine/hooks/wrong-check-reflect" "$HOME/.codex/hooks/wrong-check-reflect" +link_item "unverified-tag-check" "$REPO_DIR/engine/hooks/unverified-tag-check" "$HOME/.codex/hooks/unverified-tag-check" link_item "llm-judge" "$REPO_DIR/engine/hooks/llm-judge" "$HOME/.codex/hooks/llm-judge" link_item "hook-health" "$REPO_DIR/engine/hooks/hook-health" "$HOME/.codex/hooks/hook-health" link_item "build-the-lever" "$REPO_DIR/engine/hooks/build-the-lever" "$HOME/.codex/hooks/build-the-lever" @@ -351,6 +354,7 @@ python3 "$REPO_DIR/engine/hooks/pr-schema-gate/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/history-claim-check/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/external-claim-gate/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/wrong-check-reflect/install_claude_hook.py" +python3 "$REPO_DIR/engine/hooks/unverified-tag-check/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/llm-judge/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/hook-health/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/build-the-lever/install_claude_hook.py" @@ -396,6 +400,7 @@ python3 "$REPO_DIR/engine/hooks/scope-lock/install_cursor_hook.py" python3 "$REPO_DIR/engine/hooks/auto-pr/install_cursor_hook.py" python3 "$REPO_DIR/engine/hooks/pr-schema-gate/install_cursor_hook.py" python3 "$REPO_DIR/engine/hooks/wrong-check-reflect/install_cursor_hook.py" +python3 "$REPO_DIR/engine/hooks/unverified-tag-check/install_cursor_hook.py" python3 "$REPO_DIR/engine/hooks/llm-judge/install_cursor_hook.py" python3 "$REPO_DIR/engine/hooks/hook-health/install_cursor_hook.py" python3 "$REPO_DIR/engine/hooks/build-the-lever/install_cursor_hook.py" @@ -405,6 +410,7 @@ python3 "$REPO_DIR/engine/hooks/repeat-error-stop/install_cursor_hook.py" echo "--- codex notify (\$HOME/.codex/config.toml) ---" python3 "$REPO_DIR/engine/hooks/diu-stop/install_codex_notify.py" python3 "$REPO_DIR/engine/hooks/wrong-check-reflect/install_codex_notify.py" +python3 "$REPO_DIR/engine/hooks/unverified-tag-check/install_codex_notify.py" python3 "$REPO_DIR/engine/hooks/llm-judge/install_codex_notify.py" python3 "$REPO_DIR/engine/hooks/llm-judge/install_codex_hook.py" python3 "$REPO_DIR/engine/hooks/auto-pr/install_codex_notify.py" diff --git a/tests/test_install.py b/tests/test_install.py index 1459f54e..abb22ca4 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -534,6 +534,44 @@ def test_llm_judge_inbox_wired_for_claude_cursor_and_codex(self): cursor_stop = json.load(handle)["hooks"]["stop"] self.assertEqual(sum("llm-judge/cursor_session.py" in str(e.get("command", "")) for e in cursor_stop), 1, cursor_stop) + def test_unverified_tag_check_linked_and_wired_for_all_harnesses(self): + config_path = os.path.join(self.fake_home, ".codex", "config.toml") + with open(config_path, "w") as handle: + handle.write('model = "gpt-5"\n') + result = run_install(self.fake_home) + self.assertEqual(result.returncode, 0, result.stderr) + + for agent_dir in (".claude", ".cursor", ".codex"): + target = os.path.join(self.fake_home, agent_dir, "hooks", "unverified-tag-check") + self.assertTrue(os.path.islink(target), target) + self.assertEqual(os.readlink(target), hook_src("unverified-tag-check")) + + claude_stop = self._claude_hook_commands("Stop") + self.assertEqual( + sum("unverified-tag-check/claude_stop_check.py" in command for command in claude_stop), + 1, + claude_stop, + ) + + with open(os.path.join(self.fake_home, ".cursor", "hooks.json")) as handle: + cursor_stop = json.load(handle)["hooks"]["stop"] + self.assertEqual( + sum("unverified-tag-check/cursor_session.py" in str(entry.get("command", "")) for entry in cursor_stop), + 1, + cursor_stop, + ) + + with open(config_path) as handle: + config_text = handle.read() + match = re.search(r"^notify = (\[.*\])$", config_text, re.MULTILINE) + self.assertIsNotNone(match, config_text) + notify = json.loads(match.group(1)) + self.assertEqual( + sum("unverified-tag-check/codex_notify.py" in str(item) for item in notify), + 1, + notify, + ) + def test_cursor_hooks_json_seeded_as_real_file(self): target = os.path.join(self.fake_home, ".cursor", "hooks.json") self.assertTrue(os.path.exists(target))