Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions engine/skills/reflect/scripts/tests/test_token_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,45 @@ def test_repeated_same_class_complaints_flag_frustration_and_intervention(self):
finally:
os.unlink(path)

def test_cheap_way_out_twice_flags_intervention(self):
"""The shape the detector used to score zero: no profanity, no
told-you, no accusation. The user names the shortcut the agent took,
then names it again about a different artifact, and types the
fix-the-process command."""
lines = [
codex_response_item("user", "since you took the cheap way out and needed a stop hook, add that hook and /reflect", ts="2026-09-09T04:40:47.000Z"),
codex_response_item("assistant", "Adding the hook now.", ts="2026-09-09T04:41:00.000Z"),
codex_response_item("user", "why would you submit a rule that would ignore files straight up??? thats wierd", ts="2026-09-09T05:46:08.000Z"),
codex_response_item("assistant", "Fixing the skip.", ts="2026-09-09T05:47:00.000Z"),
]
path = write_jsonl(lines)
try:
with redirect_stdout(io.StringIO()):
result = token_audit.audit_codex(path)
kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]}
self.assertIn("cheap-way-out", kinds)
self.assertIn("explicit-invocation", kinds)
flags = {fl["name"]: fl for fl in result["flags"]}
self.assertEqual(flags["intervention-must-automate"]["value"], "yes")
finally:
os.unlink(path)

def test_product_blame_and_plain_questions_do_not_flag_intervention(self):
lines = [
codex_response_item("user", "the UI is messed up on the workers page", ts="2026-09-09T04:40:47.000Z"),
codex_response_item("assistant", "Looking at the panel now.", ts="2026-09-09T04:41:00.000Z"),
codex_response_item("user", "what would you recommend for the deploy window?", ts="2026-09-09T05:46:08.000Z"),
codex_response_item("assistant", "Three options below.", ts="2026-09-09T05:47:00.000Z"),
]
path = write_jsonl(lines)
try:
with redirect_stdout(io.StringIO()):
result = token_audit.audit_codex(path)
flags = {fl["name"]: fl for fl in result["flags"]}
self.assertEqual(flags["intervention-must-automate"]["value"], "no")
finally:
os.unlink(path)

def test_committed_codex_fixture_hits_frustration_and_intervention(self):
result = token_audit.audit_codex(fixture("codex_thrash_session.jsonl"))
flags = {fl["name"]: fl for fl in result["flags"]}
Expand Down
3 changes: 3 additions & 0 deletions engine/skills/reflect/scripts/token_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ def _direct_run_targets(command):
r"|\bis the proof att?ach?ed\b"
r"|\bwhere'?s the proof\b"
r"|\bprove (to me )?that (it|this|that|the|#?\d)", re.I)),
("cheap-way-out", re.compile(r"\bcheap way out\b|\bwhy would you\b|\bthat'?s (weird|wierd)\b|\bstraight up\b", re.I)),
("explicit-invocation", re.compile(r"(?:^|\s)/(?:automate-me|reflect|thrash)\b", re.I)),
]

# Same-type user intervention. One correction can be cheap. Repeating the
Expand All @@ -150,6 +152,7 @@ def _direct_run_targets(command):
# blame ("the ui is messed up") does not match agent-blame.
INTERVENTION_KINDS = frozenset({
"told-you", "accusation", "agent-blame", "restated-ask", "proof-challenge",
"cheap-way-out", "explicit-invocation",
})

# function_call_output / custom_tool_call_output payloads carry their exit
Expand Down
Loading