From 7eb51690eb551ec31c428e65a83d1df26f5b3ca0 Mon Sep 17 00:00:00 2001 From: Jonathan Borduas Date: Sun, 6 Sep 2026 09:00:37 +0100 Subject: [PATCH 1/2] =?UTF-8?q?discriminates:=20refuse=20a=20flag=20alongs?= =?UTF-8?q?ide=20--self-test,=20with=20exit=2064=20=E2=80=94=20NOT=20a=20v?= =?UTF-8?q?erdict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilt on current main (which now carries #611's CONTROL FAILED diagnosis), so CI can say WHICH assertion fails on Linux. Content identical to 7379984. ⚠ THIS PR IS EXPECTED TO GO RED, and that is the finding. On main this tool is UNVERIFIABLE, which PREEMPTS the control verdict — so the gate has never run its control. Repairing the flag surface makes the gate reach it, and it FAILS on CI/Linux while passing on macOS. ⇒ UNVERIFIABLE was masking a genuinely failing control. #611 was landed first so this run reports which assertion, rather than only that one did. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DTMf4EvaTEnDKXY47efRZZ --- tools/discriminates.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tools/discriminates.py b/tools/discriminates.py index 9426861..5c81283 100755 --- a/tools/discriminates.py +++ b/tools/discriminates.py @@ -60,6 +60,12 @@ Exit: 0 the check discriminated · 2 non-discriminating (verdict refused) 3 the control failed — the harness itself is broken 4 a state is not self-consistent — the comparison is uninterpretable + 64 USAGE — an unrecognised argument. ⛔ NOT a verdict, and deliberately outside + 0/2/3/4: every one of those is a statement ABOUT THE COMPARISON, and a + mistyped flag is a statement about the INVOCATION. 64 is EX_USAGE (BSD + sysexits), chosen because this file's verdict space was already full and + reusing 2 would have made "you typed it wrong" indistinguishable from + "the check did not discriminate" — the exact exit-2 collision #58 records. """ import argparse, os, subprocess, sys @@ -184,6 +190,21 @@ def main(): # documented NON-DISCRIMINATING verdict. A control that cannot be invoked without # emitting a real verdict code is worse than no control (#58, the exit-2 collision). if "--self-test" in sys.argv: + # ⛔ AN UNRECOGNISED FLAG ALONGSIDE `--self-test` MUST REFUSE. The gate measured this + # file UNVERIFIABLE: `--self-test --zzz-not-a-flag` exited 0, so "the flag is matched + # and the rest DISCARDED — a control result here describes an invocation that was only + # half read." Measured 2026-09-06. + # ⚠ 64, not 2. This file's 0/2/3/4 are ALL verdicts about the comparison; a mistyped + # flag is not one. Reusing 2 would make it indistinguishable from NON-DISCRIMINATING, + # which is the collision the interception below already exists to avoid (#58). + _extra = [a for a in sys.argv[1:] if a != "--self-test"] + if _extra: + print(f"⛔ USAGE — unrecognised argument(s) alongside --self-test: {_extra}.\n" + f" The controls take no other flags, and running them while ignoring an\n" + f" argument would report a verdict for an invocation that was only half read.\n" + f" NO REMEDY — run `--self-test` alone. (exit 64 = usage, NOT a verdict)", + file=sys.stderr) + return 64 return self_test() args = ap.parse_args() From f35c886569a48c858922fa0532b4918ba953530f Mon Sep 17 00:00:00 2001 From: Jonathan Borduas Date: Sun, 6 Sep 2026 09:03:50 +0100 Subject: [PATCH 2/2] discriminates: the self-inconsistent control used $RANDOM, which dash does not have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI/Linux said, via #611's new diagnosis: ⛔ discriminates.py CONTROL FAILED (exit 1) ── its own output, the failing lines only ── FAIL known-negative a self-inconsistent state -> 4: got 0 want 4 ⇒ `run()` uses shell=True, which is `/bin/sh` — bash on macOS, usually DASH on Linux. DASH HAS NO `$RANDOM`. It expands to EMPTY, so `echo $RANDOM$RANDOM$RANDOM` prints the same blank line twice, the two reads AGREE, and this tool CORRECTLY reports a stable state — exit 0 against an expected 4. ★ THE TOOL WAS RIGHT AND THE FIXTURE WAS NOT PORTABLE. Measured, two-poled: /bin/sh (macOS, bash) echo $RANDOM -> 29547 dash echo $RANDOM -> '' (empty) OLD fixture under dash: two calls DIFFER = NO ⇐ why CI failed NEW fixture under dash: two calls DIFFER = YES ⇐ the control can fire ⇒ `od -An -N8 -tx1 /dev/urandom` exists on both and differs per invocation. Control on the other side: `echo SAME` does not differ. ⛔ AND WHY IT SURVIVED TWO WEEKS. This tool's flag surface was UNVERIFIABLE — `--self-test --zzz-not-a-flag` exited 0 — and UNVERIFIABLE PREEMPTS the control verdict. So the gate had never run this control on Linux at all. Repairing the argument surface is what made it reachable; the failure was not introduced, it was UNCOVERED. ⇒ Three changes had to land in order for this to be findable: #608 the flag surface discriminates -> the gate REACHES the control #611 CONTROL FAILED prints its output -> the gate SAYS WHICH assertion this the fixture is portable -> the control passes for a reason ⚠ I could not reproduce it locally: the control passes on macOS before AND after. Every step of the diagnosis came from CI's own output, which is exactly what #611 was landed to make possible. test_discriminates 0 · --self-test 0, all four documented exits reachable. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DTMf4EvaTEnDKXY47efRZZ --- tools/discriminates.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/discriminates.py b/tools/discriminates.py index 5c81283..1b2ac80 100755 --- a/tools/discriminates.py +++ b/tools/discriminates.py @@ -160,9 +160,20 @@ def rc(*args): "--control-a", "echo X", "--control-b", "echo Y"), 0) # 4 — UNSTABLE: a state that differs from itself supports no comparison. - # $RANDOM is re-evaluated per invocation, so the two reads disagree. + # ⛔ NOT `$RANDOM`. `run()` uses shell=True, which is `/bin/sh` — bash on macOS, + # usually DASH on Linux, and dash HAS NO `$RANDOM`. There it expands to EMPTY, so + # `echo $RANDOM$RANDOM$RANDOM` prints the same blank line twice, the two reads AGREE, + # and this tool CORRECTLY reports a stable state — exit 0 against an expected 4. + # ⇒ The tool was right and the fixture was not portable. Measured 2026-09-06: + # /bin/sh (macOS, bash) echo $RANDOM -> 29547 + # dash echo $RANDOM -> '' (empty) + # ⚠ It passed on macOS for two weeks and failed on CI/Linux the moment the gate could + # reach it at all — the flag surface had been UNVERIFIABLE, which preempts the control + # verdict, so nothing had ever run this control on Linux. + # ⇒ /dev/urandom is present on both and differs per invocation. Verified two-poled: + # `od -An -N8 -tx1 /dev/urandom` differs across calls; `echo SAME` does not. check("known-negative a self-inconsistent state -> 4 (uninterpretable)", - rc("--a", "echo $RANDOM$RANDOM$RANDOM", "--b", "echo B"), 4) + rc("--a", "od -An -N8 -tx1 /dev/urandom", "--b", "echo B"), 4) # ⚠ Exit status is part of a reading, not just stdout. A command that fails with # empty output must not read as one that succeeds with empty output.