merge queue: checking #518 on main (0d0daba), stacked on #470 and #483

engine/hooks/hedge-runs-prove-it/README.md

@@ -48,6 +48,20 @@ Stays silent on a diagnosis inside a fence, a double-quoted span, a backtick
 span, a markdown blockquote, a hypothetical ("if it's a zombie, ..."), and
 past-tense narration of an old incident ("the task was a zombie").
 
+## Capability lists copied from errors
+
+An error message can expose a local fallback or hardcoded enumeration without
+proving the capabilities of the system named in that error. The hook records
+values in bracketed or comma-joined lists from error-shaped tool results. It
+blocks an outgoing reply that repeats at least two of those values within 200
+characters of `accepts`, `supports`, `known models`, `valid`, or `only` when
+those values appeared nowhere in a non-error tool result.
+
+One repeated value stays silent to avoid collisions on common tokens. The rule
+also stays silent when a non-error tool result supplies the repeated values, or
+when the reply attributes or retracts the enumeration with wording such as
+`fallback`, `hardcoded`, `built-in`, `in the error`, or `retract`.
+
 Mechanical half of `corpus/skills/cat-mode/SKILL.md`'s Verify rule:
 "Unhedged root-cause or fix claims about live system behavior need
 instrument-level proof in the same message, or `{{CAT-UNVERIFIED}}`." Four
@@ -73,7 +87,8 @@ escape hatch, not a free pass). Fail-open on parse or read errors;
 
 ## Files
 
-- `detect.py` -- hedge, code-noun, and reason patterns; turn scan; `decide()`.
+- `detect.py` -- hedge, diagnosis, and error-only capability-list patterns;
+  turn scan; `decide()`.
 - `claude_stop_check.py` -- Claude Stop entrypoint.
 - `claude.hook.json` / `install_claude_hook.py` -- settings.json merge (idempotent).
 - `tests/fixtures/hedges_{fires,silent}.json` -- sanitized real replies and

engine/hooks/hedge-runs-prove-it/detect.py

@@ -1,6 +1,6 @@
-"""hedge-runs-prove-it: an unrun check about code or repo state is a prompt to verify.
+"""hedge-runs-prove-it: unproven code, state, and capability claims must be verified.
 
-Two shapes, two bars.
+Three shapes, three bars.
 
 A hedge -- "I think", "I believe", "probably", "should work", "presumably",
 or a `{{CAT-UNVERIFIED}}` tag next to a code noun (a path, a backticked name,
@@ -18,6 +18,11 @@
 Only instrument-level proof in the same message clears it: pasted output, a
 `file:line`, a pid, an exit code, or an explicit `{{CAT-UNVERIFIED}}` tag.
 
+A capability enumeration copied from error-shaped tool output is not proof of
+what another system accepts or supports. A reply that repeats two or more of
+those values beside a capability verb is blocked unless a non-error tool result
+also supplied them, or the reply attributes the list as fallback/error data.
+
 Hedges about things that are not code or state (a company's motive, a
 filing date) are out of scope, and so is either shape quoted rather than
 claimed -- anywhere inside a double-quoted, backticked or single-quoted run,
@@ -118,6 +123,36 @@
     "file:line. Otherwise tag the claim: `{tag}`."
 )
 
+ERROR_OUTPUT_RE = re.compile(
+    r"\b(?:error|exception|fatal|failure|failed|invalid|unsupported|traceback)\b|"
+    r"\bnot\s+supported\b",
+    re.IGNORECASE,
+)
+BRACKETED_ENUM_RE = re.compile(r"\[([^\[\]\n]+)\]")
+COMMA_ENUM_RE = re.compile(
+    r"(?<![A-Za-z0-9_.+/@:-])"
+    r"([A-Za-z0-9][A-Za-z0-9_.+/@:-]*(?:\s*,\s*[A-Za-z0-9][A-Za-z0-9_.+/@:-]*)+)"
+    r"(?![A-Za-z0-9_.+/@:-])"
+)
+ENUM_VALUE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.+/@:-]*$")
+CAPABILITY_RE = re.compile(
+    r"\b(?:accepts?|supports?|known\s+models?|valid|only)\b",
+    re.IGNORECASE,
+)
+CAPABILITY_ATTRIBUTION_RE = re.compile(
+    r"\b(?:fallback|hardcoded|hard-coded|built[ -]in|retract(?:ed|ing|ion)?)\b|"
+    r"\b(?:in|from)\s+(?:the|this|that|an)\s+error\b|"
+    r"\b(?:the|this|that)\s+error\s+(?:says|lists|reported|showed)\b",
+    re.IGNORECASE,
+)
+VALUE_CHARS = "A-Za-z0-9_.+/@:-"
+
+CAPABILITY_MESSAGE = (
+    "hedge-runs-prove-it: this reply asserts capability values copied only from "
+    "error-shaped tool output ({values}). Verify them from a non-error source, or "
+    "attribute them as an error, fallback, hardcoded, or built-in list."
+)
+
 
 def _sentence_after(text: str, start: int) -> str:
     end = len(text)
@@ -215,6 +250,80 @@ def _is_human_user_line(data: dict) -> bool:
     return bool(text.strip()) and not text.lstrip().startswith("<")
 
 
+def _tool_result_text(block: dict) -> str:
+    content = block.get("content")
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return ""
+    chunks: list[str] = []
+    for item in content:
+        if isinstance(item, str):
+            chunks.append(item)
+        elif isinstance(item, dict) and isinstance(item.get("text"), str):
+            chunks.append(item["text"])
+    return "\n".join(chunks)
+
+
+def _tool_results(lines: list[dict]):
+    for data in lines:
+        message = data.get("message")
+        content = message.get("content") if isinstance(message, dict) else data.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_result":
+                yield block, _tool_result_text(block)
+
+
+def _enumerated_values(text: str) -> set[str]:
+    runs = [match.group(1) for match in BRACKETED_ENUM_RE.finditer(text or "")]
+    runs.extend(match.group(1) for match in COMMA_ENUM_RE.finditer(text or ""))
+    values: set[str] = set()
+    for run in runs:
+        for raw_value in run.split(","):
+            value = raw_value.strip().strip("'\"`")
+            if len(value) >= 2 and ENUM_VALUE_RE.fullmatch(value):
+                values.add(value.lower())
+    return values
+
+
+def _value_occurs(text: str, value: str) -> bool:
+    return bool(re.search(
+        rf"(?<![{VALUE_CHARS}]){re.escape(value)}(?![{VALUE_CHARS}])",
+        text or "",
+        re.IGNORECASE,
+    ))
+
+
+def error_only_capability_values(lines: list[dict]) -> set[str]:
+    """Enumerated values whose tool-result sources are all error-shaped."""
+    error_values: set[str] = set()
+    non_error_results: list[str] = []
+    for block, text in _tool_results(lines):
+        error_shaped = bool(block.get("is_error") or ERROR_OUTPUT_RE.search(text))
+        if error_shaped:
+            error_values.update(_enumerated_values(text))
+        else:
+            non_error_results.append(text)
+    return {
+        value for value in error_values
+        if not any(_value_occurs(text, value) for text in non_error_results)
+    }
+
+
+def _capability_feedback(message: str, lines: list[dict]) -> str | None:
+    if not CAPABILITY_RE.search(message or "") or CAPABILITY_ATTRIBUTION_RE.search(message or ""):
+        return None
+    values = error_only_capability_values(lines)
+    for verb in CAPABILITY_RE.finditer(message):
+        window = message[max(0, verb.start() - PROXIMITY): verb.end() + PROXIMITY]
+        repeated = sorted(value for value in values if _value_occurs(window, value))
+        if len(repeated) >= 2:
+            return CAPABILITY_MESSAGE.format(values=", ".join(repeated[:4]))
+    return None
+
+
 def parse_lines(raw_lines) -> list[dict]:
     parsed: list[dict] = []
     for raw in raw_lines:
@@ -257,11 +366,12 @@ def decide_from_lines(message: str, lines: list[dict]) -> str | None:
     if diagnosis:
         return diagnosis
     hedges = code_hedges(message)
-    if not hedges:
-        return None
-    if verified_this_turn(lines):
-        return None
-    return MESSAGE.format(tag=markers.TAG_TEMPLATE, hedge=", ".join(f'"{h}"' for h in hedges[:3]))
+    if hedges and not verified_this_turn(lines):
+        return MESSAGE.format(
+            tag=markers.TAG_TEMPLATE,
+            hedge=", ".join(f'"{h}"' for h in hedges[:3]),
+        )
+    return _capability_feedback(message, lines)
 
 
 def decide(payload: dict) -> str | None:
@@ -272,7 +382,7 @@ def decide(payload: dict) -> str | None:
     diagnosis = _diagnosis_feedback(message)
     if diagnosis:
         return diagnosis
-    if not code_hedges(message):
+    if not code_hedges(message) and not CAPABILITY_RE.search(message):
         return None
     transcript_path = payload.get("transcript_path") or payload.get("transcriptPath") or ""
     lines: list[dict] = []

engine/hooks/hedge-runs-prove-it/tests/test_hooks.py

@@ -43,6 +43,35 @@ def turn_lines(verified):
     return lines
 
 
+ERROR_CAPABILITY_LIST = (
+    'Error: Execution model "gpt-5.6-luna" is not supported for execution agent '
+    '"codex". Known models: [gpt-5.5, gpt-5.5-pro, gpt-5.4, gpt-5.4-pro].'
+)
+
+
+def capability_lines(with_non_error_source=False):
+    lines = turn_lines(False) + [
+        {"type": "assistant", "message": {"role": "assistant", "content": [
+            {"type": "tool_use", "id": "models-error", "name": "Task", "input": {}}
+        ]}},
+        {"type": "user", "message": {"role": "user", "content": [
+            {"type": "tool_result", "tool_use_id": "models-error", "content": ERROR_CAPABILITY_LIST}
+        ]}},
+    ]
+    if with_non_error_source:
+        lines.extend([
+            {"type": "assistant", "message": {"role": "assistant", "content": [
+                {"type": "tool_use", "id": "models-answer", "name": "Task", "input": {}}
+            ]}},
+            {"type": "user", "message": {"role": "user", "content": [
+                {"type": "tool_result", "tool_use_id": "models-answer", "content": (
+                    "Supported execution models: gpt-5.5, gpt-5.5-pro, gpt-5.4, gpt-5.4-pro."
+                )}
+            ]}},
+        ])
+    return lines
+
+
 def transcript_file(lines):
     tmp = tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False, encoding="utf-8")
     tmp.write("\n".join(json.dumps(line) for line in lines) + "\n")
@@ -218,5 +247,37 @@ def test_diagnosis_gate_ignores_an_unreadable_transcript(self):
         }))
 
 
+class TestBlocksCapabilitiesCopiedOnlyFromErrors(unittest.TestCase):
+    def test_hook_blocks_two_values_repeated_beside_only_accepts(self):
+        path = transcript_file(capability_lines())
+        try:
+            code, err = run_hook({
+                "last_assistant_message": "codex only accepts [gpt-5.5, gpt-5.5-pro].",
+                "transcript_path": path,
+            })
+        finally:
+            os.unlink(path)
+        self.assertEqual(code, 2)
+        self.assertIn("error-shaped tool output", err)
+
+    def test_allows_values_also_listed_by_non_error_tool_result(self):
+        self.assertIsNone(detect.decide_from_lines(
+            "codex only accepts [gpt-5.5, gpt-5.5-pro].",
+            capability_lines(with_non_error_source=True),
+        ))
+
+    def test_allows_attributed_fallback_list(self):
+        self.assertIsNone(detect.decide_from_lines(
+            "The built-in fallback list says codex only accepts [gpt-5.5, gpt-5.5-pro].",
+            capability_lines(),
+        ))
+
+    def test_allows_a_single_repeated_value(self):
+        self.assertIsNone(detect.decide_from_lines(
+            "codex only accepts gpt-5.5.",
+            capability_lines(),
+        ))
+
+
 if __name__ == "__main__":
     unittest.main()