From 3dfc0cbb852726a32b4422ca73d6a523400e5a99 Mon Sep 17 00:00:00 2001 From: 1339190177 <1339190177@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:30:22 +0800 Subject: [PATCH] fix(harness): stop masking lead lane failures as population binding mismatches When the lead analyzer failed for its own reason, populationDiagnostics reconciled lead-side counts from absent data, reported a conflict, and collectEvidenceBundle replaced the lane with a generic SESSION_POPULATION_BINDING_MISMATCH envelope. The bundle still failed closed, but the real failure code never reached the reader, which left reports like #164 undiagnosable. An unobserved lead lane now keeps its own failure on the lane. Binding reconciliation skips lead-attributed checks through a new optional leadObserved flag on validateSessionPopulationBundle (default true, so existing callers keep their behavior), the diagnostics record leadObserved and expose the concrete reconciliation errors array on genuine conflicts, and the fail-closed SESSION_POPULATION_BINDING_MISMATCH downgrade still applies whenever the conflict was observed against lead data. Validated with a new regression test that was verified red on the pre-fix implementation, the extended conflict assertions, and the full root suite (1771 tests). Spec: docs/specs/2026-09-23-164-lead-failure-passthrough.md, story #164. Co-authored-by: ZCode (GLM 5.3) --- ...2026-09-23-164-lead-failure-passthrough.md | 71 +++++++++++++++++++ .../evidence-bundle/index.mjs | 12 ++-- .../session-analysis/session-population.mjs | 10 +-- .../better-harness-evidence-bundle.test.mjs | 17 +++++ 4 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 docs/specs/2026-09-23-164-lead-failure-passthrough.md diff --git a/docs/specs/2026-09-23-164-lead-failure-passthrough.md b/docs/specs/2026-09-23-164-lead-failure-passthrough.md new file mode 100644 index 00000000..e842e443 --- /dev/null +++ b/docs/specs/2026-09-23-164-lead-failure-passthrough.md @@ -0,0 +1,71 @@ +# Lead lane failures must not be relabeled as session population binding mismatches + +## Traceability + +- Spec ID: 2026-09-23-164-lead-failure-passthrough +- Story: QoderAI/better-harness#164 +- Status: Implemented + +## Intent + +When the lead analyzer fails for its own reason (for example a +provider-specific throw inside `analyzeHarnessEvidence`), the evidence bundle +currently replaces that lane with a generic +`SESSION_POPULATION_BINDING_MISMATCH` envelope. The replacement happens because +`populationDiagnostics` reconciles lead-side counts from absent data +(`lead.data` is missing, so every lead count compares as `-1`), reports +`conflict`, and `collectEvidenceBundle` then overwrites the lead lane — even +though nothing about the session population binding actually conflicted. The +bundle still fails closed, but the user-visible failure code points at the +binding contract instead of the real cause, which makes reports such as #164 +undiagnosable: the original lead error is discarded before it can reach the +reader. + +An unobserved lead lane must keep its own failure on the lane, and binding +reconciliation must only judge the lead when the lead produced data. Genuine +binding conflicts (lead data present but contradicting the frozen population) +keep the existing fail-closed code and surface the concrete reconciliation +errors in the diagnostics payload. + +## Acceptance scenarios + +- AC-1: when the lead lane fails before producing data, the bundle keeps the + lane's original error code, the session population binding diagnostics report + `bound` with `leadObserved: false`, and the bundle status remains `failed`. +- AC-2: when the lead lane returns data that contradicts the frozen population + (for example public eligible counts that do not match the binding), the bundle + keeps the fail-closed `SESSION_POPULATION_BINDING_MISMATCH` lead downgrade and + the diagnostics include the concrete reconciliation error strings plus + `leadObserved: true`. +- AC-3: `validateSessionPopulationBundle` gains an optional `leadObserved` + flag (default `true`); lead-attributed structural checks are skipped only when + the flag reports the lead lane as unobserved. Existing callers keep their + behavior. + +## Non-goals + +- Changing the fail-closed contract of the binding reconciliation itself. +- Changing lane envelope shapes, fingerprints, discovery, qualification, or + provider behavior. +- Diagnosing the remaining provider-specific lead failure that #164 reports on + Windows; this change only stops masking it. + +## Plan and tasks + +1. `populationDiagnostics` computes `leadObserved`, skips lead-attributed + validation and lead public-count checks when the lead lane produced no data, + and records `leadObserved` plus the concrete `errors` array on conflict. +2. `collectEvidenceBundle` reassigns the lead lane to the binding-mismatch + envelope only when the conflict was observed against available lead data. +3. `validateSessionPopulationBundle` accepts `leadObserved` and skips + lead-attributed structural checks when the lead is unobserved. + +## Test and review evidence + +- New regression test `lead lane keeps its own failure code instead of a + fabricated binding mismatch` fails on the pre-fix implementation (verified by + stashing the source change) and passes after it. +- The existing conflict tests are extended with `leadObserved` and `errors` + assertions and keep passing unchanged in their fail-closed expectations. +- `npx vitest run test/reporting/better-harness-evidence-bundle.test.mjs + test/sessions/session-population.test.mjs` — 42 tests pass. diff --git a/scripts/harness-analysis/evidence-bundle/index.mjs b/scripts/harness-analysis/evidence-bundle/index.mjs index e31cb837..4571152c 100644 --- a/scripts/harness-analysis/evidence-bundle/index.mjs +++ b/scripts/harness-analysis/evidence-bundle/index.mjs @@ -71,11 +71,13 @@ function populationDiagnostics(population, sessionEvidence, lead) { admission: sessionEvidence.data.admissionBinding, } : null; + const leadObserved = Boolean(lead?.data); const leadBinding = lead?.data?.sessionBinding ?? null; const errors = validateSessionPopulationBundle({ population: population.binding, session, lead: leadBinding, + leadObserved, }); const sessionEligibleCount = Number(sessionEvidence?.data?.scope?.eligibleSessions ?? -1); const sessionSelectedCount = Number(sessionEvidence?.data?.scope?.selectedSessions ?? -1); @@ -84,8 +86,9 @@ function populationDiagnostics(population, sessionEvidence, lead) { || sessionSelectedCount !== session?.selection?.selected?.count) { errors.push("Session public counts do not match its population binding"); } - if (Number(leadSelection.eligibleCount ?? -1) !== population.binding.eligible.count - || Number(leadSelection.analyzedCount ?? -1) !== leadBinding?.selection?.selected?.count) { + if (leadObserved + && (Number(leadSelection.eligibleCount ?? -1) !== population.binding.eligible.count + || Number(leadSelection.analyzedCount ?? -1) !== leadBinding?.selection?.selected?.count)) { errors.push("lead public counts do not match its population binding"); } const sessionAdmission = session?.admission ?? {}; @@ -97,6 +100,7 @@ function populationDiagnostics(population, sessionEvidence, lead) { population: population.binding, sessionSelection: session?.selection ?? null, leadSelection: leadBinding?.selection ?? null, + leadObserved, episodes: { comparison: comparable ? "comparable" : "not-comparable-selection-or-policy", sessionTaskEpisodes: Number(sessionAdmission.taskEpisodes ?? 0), @@ -104,7 +108,7 @@ function populationDiagnostics(population, sessionEvidence, lead) { leadRetainedEpisodes: Number(leadAdmission.retainedTaskEpisodes ?? 0), leadZeroSignalDiscardedEpisodes: Number(leadAdmission.zeroSignalDiscardedEpisodes ?? 0), }, - ...(errors.length > 0 ? { errorCodes: ["SESSION_POPULATION_BINDING_MISMATCH"] } : {}), + ...(errors.length > 0 ? { errorCodes: ["SESSION_POPULATION_BINDING_MISMATCH"], errors } : {}), }; } @@ -149,7 +153,7 @@ export async function collectEvidenceBundle(options = {}, dependencies = {}) { leadPromise, ]); const sessionPopulationBinding = populationDiagnostics(sessionPopulation, sessionEvidence, lead); - if (sessionPopulationBinding.status === "conflict") { + if (sessionPopulationBinding.status === "conflict" && laneIsAvailable(lead)) { lead = unavailableLane("lead-analyzer", Object.assign( new Error("Session population binding mismatch"), { code: "SESSION_POPULATION_BINDING_MISMATCH" }, diff --git a/scripts/session-analysis/session-population.mjs b/scripts/session-analysis/session-population.mjs index b9188a14..5776070f 100644 --- a/scripts/session-analysis/session-population.mjs +++ b/scripts/session-analysis/session-population.mjs @@ -245,14 +245,16 @@ function leadAdmissionErrors(admission) { return errors; } -export function validateSessionPopulationBundle({ population, session, lead } = {}) { +export function validateSessionPopulationBundle({ population, session, lead, leadObserved = true } = {}) { + // An unobserved lead lane keeps its own failure on the lane; binding + // reconciliation must not manufacture lead errors from absent data. const errors = [ ...populationErrors(population, "Session", session?.population), - ...populationErrors(population, "lead", lead?.population), + ...(leadObserved ? populationErrors(population, "lead", lead?.population) : []), ...selectionErrors(population, "Session", session?.selection), - ...selectionErrors(population, "lead", lead?.selection), + ...(leadObserved ? selectionErrors(population, "lead", lead?.selection) : []), ...sessionAdmissionErrors(session?.admission), - ...leadAdmissionErrors(lead?.admission), + ...(leadObserved ? leadAdmissionErrors(lead?.admission) : []), ]; if (session?.admission?.projectionPolicyFingerprint !== session?.selection?.projectionPolicyFingerprint) { errors.push("Session admission policy does not match its selection binding"); diff --git a/test/reporting/better-harness-evidence-bundle.test.mjs b/test/reporting/better-harness-evidence-bundle.test.mjs index 384fa4c8..3252fd4b 100644 --- a/test/reporting/better-harness-evidence-bundle.test.mjs +++ b/test/reporting/better-harness-evidence-bundle.test.mjs @@ -1043,6 +1043,23 @@ test("Session population conflict rejects lead counts that contradict its bindin assert.equal(result.status, "failed"); assert.equal(result.lead.error.code, "SESSION_POPULATION_BINDING_MISMATCH"); assert.equal(result.diagnostics.sessionPopulationBinding.status, "conflict"); + assert.equal(result.diagnostics.sessionPopulationBinding.leadObserved, true); + assert.ok(result.diagnostics.sessionPopulationBinding.errors + .includes("lead public counts do not match its population binding")); +}); + +test("lead lane keeps its own failure code instead of a fabricated binding mismatch", async () => { + const result = await collectEvidenceBundle({ workspace: ".", platform: "codex" }, dependencies({ + analyzeHarnessEvidence: async () => { + throw Object.assign(new Error("lead failed before binding"), { code: "LEAD_FAILED_BEFORE_BINDING" }); + }, + })); + + assert.equal(result.status, "failed"); + assert.equal(result.lead.status, "unavailable"); + assert.equal(result.lead.error.code, "LEAD_FAILED_BEFORE_BINDING"); + assert.equal(result.diagnostics.sessionPopulationBinding.status, "bound"); + assert.equal(result.diagnostics.sessionPopulationBinding.leadObserved, false); }); test("Session facts reject counts that contradict the shared all-eligible population", async () => {