Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion engine/skills/reflect/scripts/tests/test_token_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -658,7 +658,7 @@ def test_product_blame_and_plain_questions_do_not_flag_intervention(self):
lines = [
codex_response_item("user", "the UI is messed up on the workers page", ts="2026-09-09T04:40:47.000Z"),
codex_response_item("assistant", "Looking at the panel now.", ts="2026-09-09T04:41:00.000Z"),
codex_response_item("user", "what would you recommend for the deploy window?", ts="2026-09-09T05:46:08.000Z"),
codex_response_item("user", "why did you choose Saturday for the deploy window?", ts="2026-09-09T05:46:08.000Z"),
codex_response_item("assistant", "Three options below.", ts="2026-09-09T05:47:00.000Z"),
]
path = write_jsonl(lines)
Expand All @@ -670,6 +670,26 @@ def test_product_blame_and_plain_questions_do_not_flag_intervention(self):
finally:
os.unlink(path)

def test_handed_over_something_broken_flags_intervention(self):
"""The wording that scored zero: no profanity, no told-you. The user
says the artifact was bogus, then that it did not run."""
lines = [
codex_response_item("user", "huh? why did you give me a bogus script?", ts="2026-09-11T05:40:00.000Z"),
codex_response_item("assistant", "Fixing the quoting now.", ts="2026-09-11T05:41:00.000Z"),
codex_response_item("user", "it still didn't run", ts="2026-09-11T05:50:00.000Z"),
codex_response_item("assistant", "Smoke-testing the transport first.", ts="2026-09-11T05:51:00.000Z"),
]
path = write_jsonl(lines)
try:
with redirect_stdout(io.StringIO()):
result = token_audit.audit_codex(path)
kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]}
self.assertIn("cheap-way-out", kinds)
flags = {fl["name"]: fl for fl in result["flags"]}
self.assertEqual(flags["intervention-must-automate"]["value"], "yes")
finally:
os.unlink(path)

def test_committed_codex_fixture_hits_frustration_and_intervention(self):
result = token_audit.audit_codex(fixture("codex_thrash_session.jsonl"))
flags = {fl["name"]: fl for fl in result["flags"]}
Expand Down
2 changes: 1 addition & 1 deletion engine/skills/reflect/scripts/token_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ def _direct_run_targets(command):
r"|\bis the proof att?ach?ed\b"
r"|\bwhere'?s the proof\b"
r"|\bprove (to me )?that (it|this|that|the|#?\d)", re.I)),
("cheap-way-out", re.compile(r"\bcheap way out\b|\bwhy would you\b|\bthat'?s (weird|wierd)\b|\bstraight up\b", re.I)),
("cheap-way-out", re.compile(r"\bcheap way out\b|\bwhy would you\b|\bbogus\b|\bdidn'?t (even )?(work|run)\b|\bthat'?s (weird|wierd)\b|\bstraight up\b", re.I)),
("explicit-invocation", re.compile(r"(?:^|\s)/(?:automate-me|reflect|thrash)\b", re.I)),
]

Expand Down
Loading