fix(orchestration): isolate per-task config-resolution failures from the suite

pyproject.toml

@@ -63,7 +63,7 @@ dev = [
     "pytest-mock>=3.15.1",
     "pytest-cov>=7.1.0",
     "pytest-xdist[psutil]>=3.0.0",
-    "mcp>=1.26.0",
+    "mcp>=1.28.1",  # >=1.28.1 clears CVE-2026-52869/52870 (1.27.2) + CVE-2026-59950 (1.28.1)
     "ruff>=0.15.7",
     "pyright>=1.1.408",
     "pip-audit>=2.10.0",

src/coder_eval/cli/run_command.py

@@ -583,9 +583,13 @@ async def _run_with_experiment(
         default_experiment = experiment  # fall back to custom as its own baseline
 
     # Resolve tasks through experiment layer (applies all 5 config layers).
-    # Layer-5 override failures (invalid -D value/path, sdk_options on a
-    # non-claude agent, the agent.type guard) and duplicate-task-id checks raise
-    # ValueError here; surface them as a clean CLI error instead of a traceback.
+    # Global failures raise ValueError here — duplicate task IDs, early-stop
+    # arming, or an invocation error that trips every task identically (bad
+    # --type / -D value, repeats over the cap) — and we surface them as a clean
+    # CLI error instead of a traceback. Per-task config-resolution failures
+    # (e.g. sdk_options on a non-claude agent) among otherwise-resolvable tasks
+    # are NOT raised: resolve_all_tasks isolates them into `skipped` so one
+    # incompatible task can't abort the whole suite.
     try:
         resolved, skipped = resolve_all_tasks(
             task_files=all_task_files,

src/coder_eval/orchestration/experiment.py

@@ -552,10 +552,18 @@ def resolve_all_tasks(
     Also handles tag filtering and unique task ID validation.
 
     Task YAMLs that fail to load (YAML parse error, Pydantic validation,
-    dataset expansion error) are recorded in the returned ``skipped`` list
-    and excluded from the resolved set rather than aborting the suite. The
-    caller surfaces ``skipped`` in the run summary so the failure is loud
-    but recoverable.
+    dataset expansion error) are recorded in the returned ``skipped`` list and
+    excluded from the resolved set rather than aborting the suite.
+
+    Per-task config-resolution failures (a task whose own YAML is incompatible
+    with the resolved run — e.g. Claude-only ``sdk_options`` surviving a
+    ``--type codex`` override, which ``CodexAgentConfig`` forbids) are likewise
+    demoted to ``skipped`` — but only when other tasks resolve. If EVERY task
+    that reaches resolution fails, the cause is a global invocation error (a bad
+    ``--type`` / ``-D`` value, repeats over the cap) rather than a per-task
+    incompatibility, so it is re-raised and aborts the run. Early-stop arming
+    errors always propagate. The caller surfaces ``skipped`` in the run summary
+    so per-task failures are loud but recoverable.
 
     Args:
         task_files: Paths to task YAML files.
@@ -572,10 +580,18 @@ def resolve_all_tasks(
     Raises:
         ValueError: If duplicate task IDs are found after resolution.
     """
-    from .early_stop import validate_early_stop
+    from .early_stop import EarlyStopConfigError, validate_early_stop
 
     resolved: list[ResolvedTask] = []
     skipped: list[SkippedTask] = []
+    # Per-task config-resolution failures are collected here rather than raised
+    # inline. After the loop they are demoted to ``skipped`` — UNLESS every task
+    # that reached resolution failed, which signals a global invocation error
+    # (bad --type, an invalid -D value, repeats over the cap) that trips every
+    # task identically rather than a per-task incompatibility; that case is
+    # re-raised so the CLI aborts cleanly instead of producing an empty run.
+    resolution_errors: list[tuple[Path, Exception]] = []
+    attempted = 0
 
     # Resolve variant-level initial_prompt_file paths before the main loop
     exp_dir = experiment_file.parent if experiment_file is not None else None
@@ -623,50 +639,97 @@ def resolve_all_tasks(
             skipped.append(SkippedTask(path=str(task_file), reason=reason))
             continue
 
-        for expanded_task in expanded_tasks:
-            for variant in experiment.variants:
-                # Apply layers 1-4 (default → experiment-defaults → task → variant) + resolve repeats
-                resolved_task, lineage, effective_repeats = resolve_task_for_variant(
-                    default_experiment, expanded_task, experiment, variant, config
-                )
+        # Isolate layer 1-5 config resolution per task file. A task whose own
+        # YAML config is incompatible with the resolved run — e.g. Claude-only
+        # agent fields (`sdk_options`) surviving a `--type codex` override, which
+        # `CodexAgentConfig` forbids (`extra="forbid"`) — raises here. Without
+        # isolation that single task aborts the entire coder-eval run. Buffer the
+        # file's resolved tasks and commit them only once the whole file
+        # resolves, so a mid-file failure discards this file's fan-out as a unit
+        # (mirroring the load/expand isolation above) rather than leaving a
+        # partial, lopsided fan-out behind.
+        attempted += 1
+        file_resolved: list[ResolvedTask] = []
+        try:
+            for expanded_task in expanded_tasks:
+                for variant in experiment.variants:
+                    # Apply layers 1-4 (default → experiment-defaults → task → variant) + resolve repeats
+                    resolved_task, lineage, effective_repeats = resolve_task_for_variant(
+                        default_experiment, expanded_task, experiment, variant, config
+                    )
 
-                # Resolve file paths injected by variant overrides
-                resolve_task_files(resolved_task, task_file, experiment_file)
-
-                # Apply prompt mutations or overrides (between file resolution and CLI overrides)
-                _apply_prompt_overrides(resolved_task, experiment, variant, lineage)
-
-                # Apply layer 5 (CLI overrides)
-                _apply_cli_overrides(resolved_task, config, lineage)
-
-                # Early-stop guardrails: run once the task is fully resolved (all 5
-                # layers merged, incl. -D run_limits.stop_early). No-op unless armed;
-                # a bad arming raises EarlyStopConfigError (a ValueError) which the
-                # run path converts to a clean CLI error.
-                validate_early_stop(resolved_task)
-
-                # Fan-out: simulation n_trials takes precedence over experiment repeats
-                # when simulation is active; otherwise use experiment-level repeats.
-                sim = resolved_task.simulation
-                n_trials = sim.n_trials if (sim is not None and sim.enabled) else 1
-                fan_count = n_trials if n_trials > 1 else effective_repeats
-                for rep in range(fan_count):
-                    resolved.append(
-                        ResolvedTask(
-                            task=resolved_task,
-                            task_file=task_file,
-                            run_dir=build_task_run_dir(
-                                config.run_dir,
-                                variant.variant_id,
-                                resolved_task.task_id,
+                    # Resolve file paths injected by variant overrides
+                    resolve_task_files(resolved_task, task_file, experiment_file)
+
+                    # Apply prompt mutations or overrides (between file resolution and CLI overrides)
+                    _apply_prompt_overrides(resolved_task, experiment, variant, lineage)
+
+                    # Apply layer 5 (CLI overrides)
+                    _apply_cli_overrides(resolved_task, config, lineage)
+
+                    # Early-stop guardrails: run once the task is fully resolved (all 5
+                    # layers merged, incl. -D run_limits.stop_early). No-op unless armed;
+                    # a bad arming raises EarlyStopConfigError (a ValueError).
+                    validate_early_stop(resolved_task)
+
+                    # Fan-out: simulation n_trials takes precedence over experiment repeats
+                    # when simulation is active; otherwise use experiment-level repeats.
+                    sim = resolved_task.simulation
+                    n_trials = sim.n_trials if (sim is not None and sim.enabled) else 1
+                    fan_count = n_trials if n_trials > 1 else effective_repeats
+                    for rep in range(fan_count):
+                        file_resolved.append(
+                            ResolvedTask(
+                                task=resolved_task,
+                                task_file=task_file,
+                                run_dir=build_task_run_dir(
+                                    config.run_dir,
+                                    variant.variant_id,
+                                    resolved_task.task_id,
+                                    replicate_index=rep,
+                                ),
+                                variant_id=variant.variant_id,
                                 replicate_index=rep,
-                            ),
-                            variant_id=variant.variant_id,
-                            replicate_index=rep,
-                            source_yaml=source_yaml,
-                            config_lineage=dict(lineage),
+                                source_yaml=source_yaml,
+                                config_lineage=dict(lineage),
+                            )
                         )
-                    )
+        # Early-stop arming errors are a deliberate hard stop: they always
+        # propagate (never demoted to skipped) so a misarmed run fails loudly.
+        except EarlyStopConfigError:
+            raise
+        # Narrow set, matching the load/expand block above: config-resolution
+        # and IO failures are collected (decided after the loop, below);
+        # AttributeError / TypeError / ImportError still crash loudly as
+        # regressions. Pydantic ValidationError is a ValueError subclass in v2,
+        # so it's covered.
+        except (FileNotFoundError, OSError, ValueError, yaml.YAMLError) as exc:
+            resolution_errors.append((task_file, exc))
+            continue
+
+        resolved.extend(file_resolved)
+
+    # Decide the fate of collected per-task resolution failures. If EVERY task
+    # that reached resolution failed, refusing to proceed (rather than producing
+    # an empty run) is the right call. We surface the first task's own error —
+    # not a synthesized "global misconfig" message — because we can't actually
+    # tell a genuine global cause (a bad --type / -D value that trips every task
+    # identically) from N tasks each independently incompatible for the same
+    # per-task reason. A ValueError (incl. Pydantic ValidationError and the "no
+    # agent registered" guard) is re-raised verbatim so its message stays clean;
+    # only a non-ValueError (FileNotFoundError/OSError/yaml.YAMLError — e.g. a
+    # missing system_prompt_file) is normalized to ValueError so it still lands
+    # in the caller's `except ValueError` (clean typer.BadParameter) instead of
+    # escaping as a raw traceback.
+    if resolution_errors and len(resolution_errors) == attempted:
+        first_exc = resolution_errors[0][1]
+        if isinstance(first_exc, ValueError):
+            raise first_exc
+        raise ValueError(str(first_exc)) from first_exc
+    for err_file, exc in resolution_errors:
+        reason = f"{type(exc).__name__}: {exc}"[:500]
+        logger.warning("Skipping task file %s — config resolution failed: %s", err_file, reason)
+        skipped.append(SkippedTask(path=str(err_file), reason=reason))
 
     # Filter by tags
     if config.include_tags or config.exclude_tags: