Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions engine/skills/reflect/scripts/tests/test_token_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,40 @@ def test_product_blame_and_plain_questions_do_not_flag_intervention(self):
finally:
os.unlink(path)

def test_two_slash_commands_in_one_submission_are_not_a_repeat(self):
"""`/reflect /cat-mode` on one line injects the same text twice, tens
of milliseconds apart. That is one message, not a re-send."""
text = "please prevent this shit from happening again"
lines = [
codex_response_item("user", text, ts="2026-09-11T05:40:00.000Z"),
codex_response_item("user", text, ts="2026-09-11T05:40:00.041Z"),
codex_response_item("assistant", "Working on it.", ts="2026-09-11T05:40:05.000Z"),
]
path = write_jsonl(lines)
try:
with redirect_stdout(io.StringIO()):
result = token_audit.audit_codex(path)
kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]}
self.assertNotIn("verbatim-repeat", kinds)
finally:
os.unlink(path)

def test_a_real_resend_minutes_later_still_counts_as_a_repeat(self):
text = "please prevent this shit from happening again"
lines = [
codex_response_item("user", text, ts="2026-09-11T05:40:00.000Z"),
codex_response_item("assistant", "Working on it.", ts="2026-09-11T05:40:20.000Z"),
codex_response_item("user", text, ts="2026-09-11T05:44:00.000Z"),
]
path = write_jsonl(lines)
try:
with redirect_stdout(io.StringIO()):
result = token_audit.audit_codex(path)
kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]}
self.assertIn("verbatim-repeat", kinds)
finally:
os.unlink(path)

def test_handed_over_something_broken_flags_intervention(self):
"""The wording that scored zero: no profanity, no told-you. The user
says the artifact was bogus, then that it did not run."""
Expand Down
5 changes: 4 additions & 1 deletion engine/skills/reflect/scripts/token_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,8 @@ def _direct_run_targets(command):
# class (told-you / accusation / agent-blame twice, two of those kinds in
# one session, or a verbatim re-send) is the automate-me trigger. Product
# blame ("the ui is messed up") does not match agent-blame.
MIN_RESEND_GAP_SECS = 5

INTERVENTION_KINDS = frozenset({
"told-you", "accusation", "agent-blame", "restated-ask", "proof-challenge",
"cheap-way-out", "explicit-invocation",
Expand Down Expand Up @@ -299,7 +301,8 @@ def frustration_signals(user_msgs, interruptions=0, failed_turn_indices=None):
norm = re.sub(r"\s+", " ", t).casefold()
if len(norm) >= 12:
for prev_secs, prev_norm, prev_idx in seen:
if prev_norm == norm and (secs is None or prev_secs is None or 0 <= secs - prev_secs <= 600):
gap = None if (secs is None or prev_secs is None) else secs - prev_secs
if prev_norm == norm and (gap is None or MIN_RESEND_GAP_SECS <= gap <= 600):
if _failed is not None and _has_index_between(_failed, prev_idx, idx):
continue
kinds.append("verbatim-repeat")
Expand Down
Loading