From 0d53f8686e22178aadf36bc0e483f2a12009645b Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Thu, 10 Sep 2026 11:51:46 -0700 Subject: [PATCH] audit(token_audit): count naming the shortcut, and typing the fix command The same-type intervention detector matched only told-you, accusation, and agent-blame, so a session where the user twice named the shortcut the agent took, and typed /automate-me and /reflect, scored zero and reported that no automate-me was owed. Two kinds join the set: cheap-way-out ("cheap way out", "why would you", "straight up", "that's weird") and explicit-invocation (a typed /automate-me, /reflect, or /thrash). Reaching for the fix-the-process command is itself the intervention. Product blame and plain questions still score zero, with fixtures both ways. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KU2pPKob4MJ1NqjsfTNyYJ Change-Id: Ifbb2b9797be1017833d9f5e2a45b3bd06ffdad61 --- .../reflect/scripts/tests/test_token_audit.py | 39 +++++++++++++++++++ engine/skills/reflect/scripts/token_audit.py | 3 ++ 2 files changed, 42 insertions(+) diff --git a/engine/skills/reflect/scripts/tests/test_token_audit.py b/engine/skills/reflect/scripts/tests/test_token_audit.py index 9944f3bf..fdf09046 100644 --- a/engine/skills/reflect/scripts/tests/test_token_audit.py +++ b/engine/skills/reflect/scripts/tests/test_token_audit.py @@ -631,6 +631,45 @@ def test_repeated_same_class_complaints_flag_frustration_and_intervention(self): finally: os.unlink(path) + def test_cheap_way_out_twice_flags_intervention(self): + """The shape the detector used to score zero: no profanity, no + told-you, no accusation. The user names the shortcut the agent took, + then names it again about a different artifact, and types the + fix-the-process command.""" + lines = [ + codex_response_item("user", "since you took the cheap way out and needed a stop hook, add that hook and /reflect", ts="2026-09-09T04:40:47.000Z"), + codex_response_item("assistant", "Adding the hook now.", ts="2026-09-09T04:41:00.000Z"), + codex_response_item("user", "why would you submit a rule that would ignore files straight up??? thats wierd", ts="2026-09-09T05:46:08.000Z"), + codex_response_item("assistant", "Fixing the skip.", ts="2026-09-09T05:47:00.000Z"), + ] + path = write_jsonl(lines) + try: + with redirect_stdout(io.StringIO()): + result = token_audit.audit_codex(path) + kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]} + self.assertIn("cheap-way-out", kinds) + self.assertIn("explicit-invocation", kinds) + flags = {fl["name"]: fl for fl in result["flags"]} + self.assertEqual(flags["intervention-must-automate"]["value"], "yes") + finally: + os.unlink(path) + + def test_product_blame_and_plain_questions_do_not_flag_intervention(self): + lines = [ + codex_response_item("user", "the UI is messed up on the workers page", ts="2026-09-09T04:40:47.000Z"), + codex_response_item("assistant", "Looking at the panel now.", ts="2026-09-09T04:41:00.000Z"), + codex_response_item("user", "what would you recommend for the deploy window?", ts="2026-09-09T05:46:08.000Z"), + codex_response_item("assistant", "Three options below.", ts="2026-09-09T05:47:00.000Z"), + ] + path = write_jsonl(lines) + try: + with redirect_stdout(io.StringIO()): + result = token_audit.audit_codex(path) + flags = {fl["name"]: fl for fl in result["flags"]} + self.assertEqual(flags["intervention-must-automate"]["value"], "no") + finally: + os.unlink(path) + def test_committed_codex_fixture_hits_frustration_and_intervention(self): result = token_audit.audit_codex(fixture("codex_thrash_session.jsonl")) flags = {fl["name"]: fl for fl in result["flags"]} diff --git a/engine/skills/reflect/scripts/token_audit.py b/engine/skills/reflect/scripts/token_audit.py index 0004d8f0..2de08754 100644 --- a/engine/skills/reflect/scripts/token_audit.py +++ b/engine/skills/reflect/scripts/token_audit.py @@ -142,6 +142,8 @@ def _direct_run_targets(command): r"|\bis the proof att?ach?ed\b" r"|\bwhere'?s the proof\b" r"|\bprove (to me )?that (it|this|that|the|#?\d)", re.I)), + ("cheap-way-out", re.compile(r"\bcheap way out\b|\bwhy would you\b|\bthat'?s (weird|wierd)\b|\bstraight up\b", re.I)), + ("explicit-invocation", re.compile(r"(?:^|\s)/(?:automate-me|reflect|thrash)\b", re.I)), ] # Same-type user intervention. One correction can be cheap. Repeating the @@ -150,6 +152,7 @@ def _direct_run_targets(command): # blame ("the ui is messed up") does not match agent-blame. INTERVENTION_KINDS = frozenset({ "told-you", "accusation", "agent-blame", "restated-ask", "proof-challenge", + "cheap-way-out", "explicit-invocation", }) # function_call_output / custom_tool_call_output payloads carry their exit