diff --git a/docs/specs/2026-09-23-164-lead-failure-passthrough.md b/docs/specs/2026-09-23-164-lead-failure-passthrough.md new file mode 100644 index 00000000..e842e443 --- /dev/null +++ b/docs/specs/2026-09-23-164-lead-failure-passthrough.md @@ -0,0 +1,71 @@ +# Lead lane failures must not be relabeled as session population binding mismatches + +## Traceability + +- Spec ID: 2026-09-23-164-lead-failure-passthrough +- Story: QoderAI/better-harness#164 +- Status: Implemented + +## Intent + +When the lead analyzer fails for its own reason (for example a +provider-specific throw inside `analyzeHarnessEvidence`), the evidence bundle +currently replaces that lane with a generic +`SESSION_POPULATION_BINDING_MISMATCH` envelope. The replacement happens because +`populationDiagnostics` reconciles lead-side counts from absent data +(`lead.data` is missing, so every lead count compares as `-1`), reports +`conflict`, and `collectEvidenceBundle` then overwrites the lead lane — even +though nothing about the session population binding actually conflicted. The +bundle still fails closed, but the user-visible failure code points at the +binding contract instead of the real cause, which makes reports such as #164 +undiagnosable: the original lead error is discarded before it can reach the +reader. + +An unobserved lead lane must keep its own failure on the lane, and binding +reconciliation must only judge the lead when the lead produced data. Genuine +binding conflicts (lead data present but contradicting the frozen population) +keep the existing fail-closed code and surface the concrete reconciliation +errors in the diagnostics payload. + +## Acceptance scenarios + +- AC-1: when the lead lane fails before producing data, the bundle keeps the + lane's original error code, the session population binding diagnostics report + `bound` with `leadObserved: false`, and the bundle status remains `failed`. +- AC-2: when the lead lane returns data that contradicts the frozen population + (for example public eligible counts that do not match the binding), the bundle + keeps the fail-closed `SESSION_POPULATION_BINDING_MISMATCH` lead downgrade and + the diagnostics include the concrete reconciliation error strings plus + `leadObserved: true`. +- AC-3: `validateSessionPopulationBundle` gains an optional `leadObserved` + flag (default `true`); lead-attributed structural checks are skipped only when + the flag reports the lead lane as unobserved. Existing callers keep their + behavior. + +## Non-goals + +- Changing the fail-closed contract of the binding reconciliation itself. +- Changing lane envelope shapes, fingerprints, discovery, qualification, or + provider behavior. +- Diagnosing the remaining provider-specific lead failure that #164 reports on + Windows; this change only stops masking it. + +## Plan and tasks + +1. `populationDiagnostics` computes `leadObserved`, skips lead-attributed + validation and lead public-count checks when the lead lane produced no data, + and records `leadObserved` plus the concrete `errors` array on conflict. +2. `collectEvidenceBundle` reassigns the lead lane to the binding-mismatch + envelope only when the conflict was observed against available lead data. +3. `validateSessionPopulationBundle` accepts `leadObserved` and skips + lead-attributed structural checks when the lead is unobserved. + +## Test and review evidence + +- New regression test `lead lane keeps its own failure code instead of a + fabricated binding mismatch` fails on the pre-fix implementation (verified by + stashing the source change) and passes after it. +- The existing conflict tests are extended with `leadObserved` and `errors` + assertions and keep passing unchanged in their fail-closed expectations. +- `npx vitest run test/reporting/better-harness-evidence-bundle.test.mjs + test/sessions/session-population.test.mjs` — 42 tests pass. diff --git a/scripts/harness-analysis/evidence-bundle/index.mjs b/scripts/harness-analysis/evidence-bundle/index.mjs index e31cb837..4571152c 100644 --- a/scripts/harness-analysis/evidence-bundle/index.mjs +++ b/scripts/harness-analysis/evidence-bundle/index.mjs @@ -71,11 +71,13 @@ function populationDiagnostics(population, sessionEvidence, lead) { admission: sessionEvidence.data.admissionBinding, } : null; + const leadObserved = Boolean(lead?.data); const leadBinding = lead?.data?.sessionBinding ?? null; const errors = validateSessionPopulationBundle({ population: population.binding, session, lead: leadBinding, + leadObserved, }); const sessionEligibleCount = Number(sessionEvidence?.data?.scope?.eligibleSessions ?? -1); const sessionSelectedCount = Number(sessionEvidence?.data?.scope?.selectedSessions ?? -1); @@ -84,8 +86,9 @@ function populationDiagnostics(population, sessionEvidence, lead) { || sessionSelectedCount !== session?.selection?.selected?.count) { errors.push("Session public counts do not match its population binding"); } - if (Number(leadSelection.eligibleCount ?? -1) !== population.binding.eligible.count - || Number(leadSelection.analyzedCount ?? -1) !== leadBinding?.selection?.selected?.count) { + if (leadObserved + && (Number(leadSelection.eligibleCount ?? -1) !== population.binding.eligible.count + || Number(leadSelection.analyzedCount ?? -1) !== leadBinding?.selection?.selected?.count)) { errors.push("lead public counts do not match its population binding"); } const sessionAdmission = session?.admission ?? {}; @@ -97,6 +100,7 @@ function populationDiagnostics(population, sessionEvidence, lead) { population: population.binding, sessionSelection: session?.selection ?? null, leadSelection: leadBinding?.selection ?? null, + leadObserved, episodes: { comparison: comparable ? "comparable" : "not-comparable-selection-or-policy", sessionTaskEpisodes: Number(sessionAdmission.taskEpisodes ?? 0), @@ -104,7 +108,7 @@ function populationDiagnostics(population, sessionEvidence, lead) { leadRetainedEpisodes: Number(leadAdmission.retainedTaskEpisodes ?? 0), leadZeroSignalDiscardedEpisodes: Number(leadAdmission.zeroSignalDiscardedEpisodes ?? 0), }, - ...(errors.length > 0 ? { errorCodes: ["SESSION_POPULATION_BINDING_MISMATCH"] } : {}), + ...(errors.length > 0 ? { errorCodes: ["SESSION_POPULATION_BINDING_MISMATCH"], errors } : {}), }; } @@ -149,7 +153,7 @@ export async function collectEvidenceBundle(options = {}, dependencies = {}) { leadPromise, ]); const sessionPopulationBinding = populationDiagnostics(sessionPopulation, sessionEvidence, lead); - if (sessionPopulationBinding.status === "conflict") { + if (sessionPopulationBinding.status === "conflict" && laneIsAvailable(lead)) { lead = unavailableLane("lead-analyzer", Object.assign( new Error("Session population binding mismatch"), { code: "SESSION_POPULATION_BINDING_MISMATCH" }, diff --git a/scripts/session-analysis/session-population.mjs b/scripts/session-analysis/session-population.mjs index b9188a14..5776070f 100644 --- a/scripts/session-analysis/session-population.mjs +++ b/scripts/session-analysis/session-population.mjs @@ -245,14 +245,16 @@ function leadAdmissionErrors(admission) { return errors; } -export function validateSessionPopulationBundle({ population, session, lead } = {}) { +export function validateSessionPopulationBundle({ population, session, lead, leadObserved = true } = {}) { + // An unobserved lead lane keeps its own failure on the lane; binding + // reconciliation must not manufacture lead errors from absent data. const errors = [ ...populationErrors(population, "Session", session?.population), - ...populationErrors(population, "lead", lead?.population), + ...(leadObserved ? populationErrors(population, "lead", lead?.population) : []), ...selectionErrors(population, "Session", session?.selection), - ...selectionErrors(population, "lead", lead?.selection), + ...(leadObserved ? selectionErrors(population, "lead", lead?.selection) : []), ...sessionAdmissionErrors(session?.admission), - ...leadAdmissionErrors(lead?.admission), + ...(leadObserved ? leadAdmissionErrors(lead?.admission) : []), ]; if (session?.admission?.projectionPolicyFingerprint !== session?.selection?.projectionPolicyFingerprint) { errors.push("Session admission policy does not match its selection binding"); diff --git a/test/reporting/better-harness-evidence-bundle.test.mjs b/test/reporting/better-harness-evidence-bundle.test.mjs index 384fa4c8..3252fd4b 100644 --- a/test/reporting/better-harness-evidence-bundle.test.mjs +++ b/test/reporting/better-harness-evidence-bundle.test.mjs @@ -1043,6 +1043,23 @@ test("Session population conflict rejects lead counts that contradict its bindin assert.equal(result.status, "failed"); assert.equal(result.lead.error.code, "SESSION_POPULATION_BINDING_MISMATCH"); assert.equal(result.diagnostics.sessionPopulationBinding.status, "conflict"); + assert.equal(result.diagnostics.sessionPopulationBinding.leadObserved, true); + assert.ok(result.diagnostics.sessionPopulationBinding.errors + .includes("lead public counts do not match its population binding")); +}); + +test("lead lane keeps its own failure code instead of a fabricated binding mismatch", async () => { + const result = await collectEvidenceBundle({ workspace: ".", platform: "codex" }, dependencies({ + analyzeHarnessEvidence: async () => { + throw Object.assign(new Error("lead failed before binding"), { code: "LEAD_FAILED_BEFORE_BINDING" }); + }, + })); + + assert.equal(result.status, "failed"); + assert.equal(result.lead.status, "unavailable"); + assert.equal(result.lead.error.code, "LEAD_FAILED_BEFORE_BINDING"); + assert.equal(result.diagnostics.sessionPopulationBinding.status, "bound"); + assert.equal(result.diagnostics.sessionPopulationBinding.leadObserved, false); }); test("Session facts reject counts that contradict the shared all-eligible population", async () => {