fix(harness): stop masking lead lane failures as population binding mismatches

docs/specs/2026-09-23-164-lead-failure-passthrough.md

@@ -0,0 +1,71 @@
+# Lead lane failures must not be relabeled as session population binding mismatches
+
+## Traceability
+
+- Spec ID: 2026-09-23-164-lead-failure-passthrough
+- Story: QoderAI/better-harness#164
+- Status: Implemented
+
+## Intent
+
+When the lead analyzer fails for its own reason (for example a
+provider-specific throw inside `analyzeHarnessEvidence`), the evidence bundle
+currently replaces that lane with a generic
+`SESSION_POPULATION_BINDING_MISMATCH` envelope. The replacement happens because
+`populationDiagnostics` reconciles lead-side counts from absent data
+(`lead.data` is missing, so every lead count compares as `-1`), reports
+`conflict`, and `collectEvidenceBundle` then overwrites the lead lane — even
+though nothing about the session population binding actually conflicted. The
+bundle still fails closed, but the user-visible failure code points at the
+binding contract instead of the real cause, which makes reports such as #164
+undiagnosable: the original lead error is discarded before it can reach the
+reader.
+
+An unobserved lead lane must keep its own failure on the lane, and binding
+reconciliation must only judge the lead when the lead produced data. Genuine
+binding conflicts (lead data present but contradicting the frozen population)
+keep the existing fail-closed code and surface the concrete reconciliation
+errors in the diagnostics payload.
+
+## Acceptance scenarios
+
+- AC-1: when the lead lane fails before producing data, the bundle keeps the
+  lane's original error code, the session population binding diagnostics report
+  `bound` with `leadObserved: false`, and the bundle status remains `failed`.
+- AC-2: when the lead lane returns data that contradicts the frozen population
+  (for example public eligible counts that do not match the binding), the bundle
+  keeps the fail-closed `SESSION_POPULATION_BINDING_MISMATCH` lead downgrade and
+  the diagnostics include the concrete reconciliation error strings plus
+  `leadObserved: true`.
+- AC-3: `validateSessionPopulationBundle` gains an optional `leadObserved`
+  flag (default `true`); lead-attributed structural checks are skipped only when
+  the flag reports the lead lane as unobserved. Existing callers keep their
+  behavior.
+
+## Non-goals
+
+- Changing the fail-closed contract of the binding reconciliation itself.
+- Changing lane envelope shapes, fingerprints, discovery, qualification, or
+  provider behavior.
+- Diagnosing the remaining provider-specific lead failure that #164 reports on
+  Windows; this change only stops masking it.
+
+## Plan and tasks
+
+1. `populationDiagnostics` computes `leadObserved`, skips lead-attributed
+   validation and lead public-count checks when the lead lane produced no data,
+   and records `leadObserved` plus the concrete `errors` array on conflict.
+2. `collectEvidenceBundle` reassigns the lead lane to the binding-mismatch
+   envelope only when the conflict was observed against available lead data.
+3. `validateSessionPopulationBundle` accepts `leadObserved` and skips
+   lead-attributed structural checks when the lead is unobserved.
+
+## Test and review evidence
+
+- New regression test `lead lane keeps its own failure code instead of a
+  fabricated binding mismatch` fails on the pre-fix implementation (verified by
+  stashing the source change) and passes after it.
+- The existing conflict tests are extended with `leadObserved` and `errors`
+  assertions and keep passing unchanged in their fail-closed expectations.
+- `npx vitest run test/reporting/better-harness-evidence-bundle.test.mjs
+  test/sessions/session-population.test.mjs` — 42 tests pass.

scripts/harness-analysis/evidence-bundle/index.mjs

@@ -71,11 +71,13 @@ function populationDiagnostics(population, sessionEvidence, lead) {
         admission: sessionEvidence.data.admissionBinding,
       }
     : null;
+  const leadObserved = Boolean(lead?.data);
   const leadBinding = lead?.data?.sessionBinding ?? null;
   const errors = validateSessionPopulationBundle({
     population: population.binding,
     session,
     lead: leadBinding,
+    leadObserved,
   });
   const sessionEligibleCount = Number(sessionEvidence?.data?.scope?.eligibleSessions ?? -1);
   const sessionSelectedCount = Number(sessionEvidence?.data?.scope?.selectedSessions ?? -1);
@@ -84,8 +86,9 @@ function populationDiagnostics(population, sessionEvidence, lead) {
     || sessionSelectedCount !== session?.selection?.selected?.count) {
     errors.push("Session public counts do not match its population binding");
   }
-  if (Number(leadSelection.eligibleCount ?? -1) !== population.binding.eligible.count
-    || Number(leadSelection.analyzedCount ?? -1) !== leadBinding?.selection?.selected?.count) {
+  if (leadObserved
+    && (Number(leadSelection.eligibleCount ?? -1) !== population.binding.eligible.count
+      || Number(leadSelection.analyzedCount ?? -1) !== leadBinding?.selection?.selected?.count)) {
     errors.push("lead public counts do not match its population binding");
   }
   const sessionAdmission = session?.admission ?? {};
@@ -97,14 +100,15 @@ function populationDiagnostics(population, sessionEvidence, lead) {
     population: population.binding,
     sessionSelection: session?.selection ?? null,
     leadSelection: leadBinding?.selection ?? null,
+    leadObserved,
     episodes: {
       comparison: comparable ? "comparable" : "not-comparable-selection-or-policy",
       sessionTaskEpisodes: Number(sessionAdmission.taskEpisodes ?? 0),
       leadProjectedEpisodes: Number(leadAdmission.projectedEpisodes ?? 0),
       leadRetainedEpisodes: Number(leadAdmission.retainedTaskEpisodes ?? 0),
       leadZeroSignalDiscardedEpisodes: Number(leadAdmission.zeroSignalDiscardedEpisodes ?? 0),
     },
-    ...(errors.length > 0 ? { errorCodes: ["SESSION_POPULATION_BINDING_MISMATCH"] } : {}),
+    ...(errors.length > 0 ? { errorCodes: ["SESSION_POPULATION_BINDING_MISMATCH"], errors } : {}),
   };
 }
 
@@ -149,7 +153,7 @@ export async function collectEvidenceBundle(options = {}, dependencies = {}) {
     leadPromise,
   ]);
   const sessionPopulationBinding = populationDiagnostics(sessionPopulation, sessionEvidence, lead);
-  if (sessionPopulationBinding.status === "conflict") {
+  if (sessionPopulationBinding.status === "conflict" && laneIsAvailable(lead)) {
     lead = unavailableLane("lead-analyzer", Object.assign(
       new Error("Session population binding mismatch"),
       { code: "SESSION_POPULATION_BINDING_MISMATCH" },

scripts/session-analysis/session-population.mjs

@@ -245,14 +245,16 @@ function leadAdmissionErrors(admission) {
   return errors;
 }
 
-export function validateSessionPopulationBundle({ population, session, lead } = {}) {
+export function validateSessionPopulationBundle({ population, session, lead, leadObserved = true } = {}) {
+  // An unobserved lead lane keeps its own failure on the lane; binding
+  // reconciliation must not manufacture lead errors from absent data.
   const errors = [
     ...populationErrors(population, "Session", session?.population),
-    ...populationErrors(population, "lead", lead?.population),
+    ...(leadObserved ? populationErrors(population, "lead", lead?.population) : []),
     ...selectionErrors(population, "Session", session?.selection),
-    ...selectionErrors(population, "lead", lead?.selection),
+    ...(leadObserved ? selectionErrors(population, "lead", lead?.selection) : []),
     ...sessionAdmissionErrors(session?.admission),
-    ...leadAdmissionErrors(lead?.admission),
+    ...(leadObserved ? leadAdmissionErrors(lead?.admission) : []),
   ];
   if (session?.admission?.projectionPolicyFingerprint !== session?.selection?.projectionPolicyFingerprint) {
     errors.push("Session admission policy does not match its selection binding");

test/reporting/better-harness-evidence-bundle.test.mjs

@@ -1043,6 +1043,23 @@ test("Session population conflict rejects lead counts that contradict its bindin
   assert.equal(result.status, "failed");
   assert.equal(result.lead.error.code, "SESSION_POPULATION_BINDING_MISMATCH");
   assert.equal(result.diagnostics.sessionPopulationBinding.status, "conflict");
+  assert.equal(result.diagnostics.sessionPopulationBinding.leadObserved, true);
+  assert.ok(result.diagnostics.sessionPopulationBinding.errors
+    .includes("lead public counts do not match its population binding"));
+});
+
+test("lead lane keeps its own failure code instead of a fabricated binding mismatch", async () => {
+  const result = await collectEvidenceBundle({ workspace: ".", platform: "codex" }, dependencies({
+    analyzeHarnessEvidence: async () => {
+      throw Object.assign(new Error("lead failed before binding"), { code: "LEAD_FAILED_BEFORE_BINDING" });
+    },
+  }));
+
+  assert.equal(result.status, "failed");
+  assert.equal(result.lead.status, "unavailable");
+  assert.equal(result.lead.error.code, "LEAD_FAILED_BEFORE_BINDING");
+  assert.equal(result.diagnostics.sessionPopulationBinding.status, "bound");
+  assert.equal(result.diagnostics.sessionPopulationBinding.leadObserved, false);
 });
 
 test("Session facts reject counts that contradict the shared all-eligible population", async () => {