From aca599c0e427fe5f80c39fcc5d636ae10f23e579 Mon Sep 17 00:00:00 2001 From: Lilly Luo Date: Tue, 22 Sep 2026 00:06:03 +0000 Subject: [PATCH 1/4] Keep Claude route agents registered during parallel spawn --- scripts/repro_stale_claude_subagent.py | 119 +++++++++++++++++++++++++ src/ucode/smart_routing/v2.py | 95 +++++++++++--------- tests/test_claude_smart_routing_v2.py | 62 +++++++------ 3 files changed, 207 insertions(+), 69 deletions(-) create mode 100644 scripts/repro_stale_claude_subagent.py diff --git a/scripts/repro_stale_claude_subagent.py b/scripts/repro_stale_claude_subagent.py new file mode 100644 index 000000000..0dfe10cb3 --- /dev/null +++ b/scripts/repro_stale_claude_subagent.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Replay the parallel Claude agent-registration failure captured in the TUI. + +The failing session launched four top-level Agent calls concurrently. Its agent +registry contained built-in and plugin agents but none of the route agents passed +through ``--agents``. This script sends the same four concurrent hook payloads and +checks whether each selected route agent exists in that captured registry shape. + +Before the fix: + uv run python scripts/repro_stale_claude_subagent.py --registration cli --expect broken + +After the fix: + uv run python scripts/repro_stale_claude_subagent.py --registration plugin --expect fixed +""" + +from __future__ import annotations + +import argparse +import json +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +from ucode.smart_routing import routing, v2 + +AVAILABLE_MODEL = "system.ai.glm-5-3" +AVAILABLE_AGENTS = { + "Explore", + "Plan", + "general-purpose", + "model-orchestrator:explorer", + "model-orchestrator:worker", +} +TASKS = ( + "Audit anti-patterns and type safety", + "Audit architecture and complexity", + "Audit test quality", + "Audit hooks, data-fetching, and correctness", +) + + +def _plugin_agents(plugin_dir: Path) -> set[str]: + agents = set() + for agent_path in (plugin_dir / "agents").glob("*.md"): + name_line = next( + line for line in agent_path.read_text().splitlines() if line.startswith("name: ") + ) + name = json.loads(name_line.removeprefix("name: ")) + agents.add(f"{v2.CLAUDE_ROUTING_PLUGIN_NAME}:{name}") + return agents + + +def _route(task: str) -> str: + output = v2.route_claude_pre_tool_use( + { + "tool_name": "Agent", + "tool_input": { + "subagent_type": "Explore", + "prompt": task, + "model": "sonnet", + }, + }, + workspace="https://example.databricks.com", + token="test-token", + available_models=[AVAILABLE_MODEL], + ) + return output["hookSpecificOutput"]["updatedInput"]["subagent_type"] + + +def reproduce(registration: str) -> list[str]: + registered_agents = set(AVAILABLE_AGENTS) + with tempfile.TemporaryDirectory() as temp_dir: + if registration == "plugin": + plugin_dir = Path(temp_dir) / "routing-plugin" + v2._write_routed_claude_plugin(plugin_dir, [AVAILABLE_MODEL]) + registered_agents.update(_plugin_agents(plugin_dir)) + + decision = routing.RoutingDecision(model=AVAILABLE_MODEL, raw_model="glm-5-3") + with ( + patch( + "ucode.smart_routing.v2.routing.select_route", + return_value=(decision, None), + ), + ThreadPoolExecutor(max_workers=len(TASKS)) as executor, + ): + selected_agents = list(executor.map(_route, TASKS)) + + if registration == "cli": + prefix = f"{v2.CLAUDE_ROUTING_PLUGIN_NAME}:" + selected_agents = [agent.removeprefix(prefix) for agent in selected_agents] + return [agent for agent in selected_agents if agent not in registered_agents] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--registration", choices=("cli", "plugin"), default="plugin") + parser.add_argument("--expect", choices=("broken", "fixed"), default="fixed") + args = parser.parse_args() + + missing_agents = reproduce(args.registration) + if missing_agents: + print( + f"Agent type '{missing_agents[0]}' not found. " + f"Available agents: {', '.join(sorted(AVAILABLE_AGENTS))}" + ) + print(f"FAILED: {len(missing_agents)} concurrent Agent calls selected missing agents") + else: + print(f"PASS: all {len(TASKS)} concurrent Agent calls selected registered plugin agents") + + actual = "broken" if missing_agents else "fixed" + if actual != args.expect: + print(f"Expected {args.expect}, observed {actual}") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ucode/smart_routing/v2.py b/src/ucode/smart_routing/v2.py index a79c51872..3858486a2 100644 --- a/src/ucode/smart_routing/v2.py +++ b/src/ucode/smart_routing/v2.py @@ -3,6 +3,7 @@ import hashlib import json import os +import shutil import signal import socket import subprocess @@ -19,7 +20,13 @@ custom_catalog_models, custom_catalog_path, ) -from ucode.config_io import APP_DIR, read_json_safe, read_toml_safe, write_json_file +from ucode.config_io import ( + APP_DIR, + read_json_safe, + read_toml_safe, + write_json_file, + write_text_file, +) from ucode.constants import LOOPBACK_HOST from ucode.custom_oauth import custom_oauth_cli_enabled, get_custom_client_token from ucode.databricks import ( @@ -57,6 +64,7 @@ HEALTH_POLL_INTERVAL_SECONDS = 0.25 CLAUDE_ROUTE_SELECTION_TIMEOUT_S = 20.0 CLAUDE_ROUTED_AGENT_PREFIX = "ucode-route-" +CLAUDE_ROUTING_PLUGIN_NAME = "ucode-smart-routing" CLAUDE_ROUTED_AGENT_PROMPT = ( "Complete the delegated task exactly as requested. Follow the parent agent's instructions and " "return a concise report of your findings or changes." @@ -231,7 +239,7 @@ def _claude_model_overrides(model_ids: list[str]) -> dict[str, str]: return overrides -def _routed_claude_agent_name(model: str) -> str: +def _routed_claude_agent_slug(model: str) -> str: canonical = _canonical_claude_model_id(model) normalized = routing.normalize_model(canonical) safe = "".join(character if character.isalnum() else "-" for character in normalized) @@ -240,6 +248,10 @@ def _routed_claude_agent_name(model: str) -> str: return f"{CLAUDE_ROUTED_AGENT_PREFIX}{slug[:36]}-{digest}" +def _routed_claude_agent_name(model: str) -> str: + return f"{CLAUDE_ROUTING_PLUGIN_NAME}:{_routed_claude_agent_slug(model)}" + + def _routed_claude_agent_definitions(model_ids: list[str]) -> dict[str, dict[str, str]]: return { _routed_claude_agent_name(model): { @@ -251,42 +263,34 @@ def _routed_claude_agent_definitions(model_ids: list[str]) -> dict[str, dict[str } -def _with_routed_claude_agents(tool_args: list[str], model_ids: list[str]) -> list[str]: - definitions = _routed_claude_agent_definitions(model_ids) - caller_definitions: dict = {} - remaining: list[str] = [] - index = 0 - while index < len(tool_args): - arg = tool_args[index] - if arg == "--": - remaining.extend(tool_args[index:]) - break - if arg == "--agents": - if index + 1 >= len(tool_args): - raise RuntimeError("Claude's --agents option requires a JSON object.") - raw = tool_args[index + 1] - index += 2 - elif arg.startswith("--agents="): - raw = arg.partition("=")[2] - index += 1 - else: - remaining.append(arg) - index += 1 - continue - try: - parsed = json.loads(raw) - except ValueError as exc: - raise RuntimeError("Claude's --agents option must contain valid JSON.") from exc - if not isinstance(parsed, dict): - raise RuntimeError("Claude's --agents option must contain a JSON object.") - caller_definitions.update(parsed) - - collisions = definitions.keys() & caller_definitions.keys() - if collisions: - names = ", ".join(sorted(collisions)) - raise RuntimeError(f"Claude --agents names conflict with smart routing: {names}.") - combined = {**caller_definitions, **definitions} - return ["--agents", json.dumps(combined, separators=(",", ":")), *remaining] +def _write_routed_claude_plugin(plugin_dir: Path, model_ids: list[str]) -> None: + """Register exact-model agents through Claude's plugin agent registry.""" + write_json_file( + plugin_dir / ".claude-plugin" / "plugin.json", + { + "name": CLAUDE_ROUTING_PLUGIN_NAME, + "version": "1.0.0", + "description": "Launch-scoped agents for Unity Gateway smart routing.", + "author": {"name": "Databricks"}, + }, + ) + for name, definition in _routed_claude_agent_definitions(model_ids).items(): + slug = name.partition(":")[2] + write_text_file( + plugin_dir / "agents" / f"{slug}.md", + "\n".join( + [ + "---", + f"name: {json.dumps(slug)}", + f"description: {json.dumps(definition['description'])}", + f"model: {json.dumps(definition['model'])}", + "---", + "", + definition["prompt"], + "", + ] + ), + ) def _request_claude_routing_decision( @@ -467,6 +471,7 @@ def launch_claude( run_id = f"{os.getpid()}-{uuid.uuid4().hex[:8]}" socket_path = APP_DIR / f"claude-v2-{run_id}.sock" settings_path = APP_DIR / f"claude-v2-{run_id}.json" + plugin_dir = APP_DIR / f"claude-v2-{run_id}-plugin" settings, remaining = compose_settings(tool_args) hook_executable = build_auth_token_argv( @@ -493,9 +498,17 @@ def launch_claude( if route_first_prompt: sync_first_prompt_hook(settings, hook_executable) write_json_file(settings_path, settings) + _write_routed_claude_plugin(plugin_dir, model_ids) model_args = launch_model_args(remaining, launch_model) - routed_agent_args = _with_routed_claude_agents(remaining, model_ids) - argv = [binary, "--settings", str(settings_path), *model_args, *routed_agent_args] + argv = [ + binary, + "--settings", + str(settings_path), + *model_args, + "--plugin-dir", + str(plugin_dir), + *remaining, + ] if not route_first_prompt: # Subagent-only routing needs no PTY: the PreToolUse hooks ride in the @@ -508,6 +521,7 @@ def launch_claude( returncode = proc.wait() finally: settings_path.unlink(missing_ok=True) + shutil.rmtree(plugin_dir, ignore_errors=True) sys.exit(returncode) model_setting = _ClaudeModelSettingGuard(user_settings_path) @@ -534,6 +548,7 @@ def route_prompt(prompt: str) -> claude_pty.FirstPromptRoute: model_setting.restore() settings_path.unlink(missing_ok=True) socket_path.unlink(missing_ok=True) + shutil.rmtree(plugin_dir, ignore_errors=True) sys.exit(returncode) diff --git a/tests/test_claude_smart_routing_v2.py b/tests/test_claude_smart_routing_v2.py index e8530ef44..95c11c4e9 100644 --- a/tests/test_claude_smart_routing_v2.py +++ b/tests/test_claude_smart_routing_v2.py @@ -16,6 +16,16 @@ from ucode.smart_routing import claude_hooks, claude_pty, routing, v2 +def _plugin_agent_models(plugin_dir: Path) -> set[str]: + models = set() + for agent_path in (plugin_dir / "agents").glob("*.md"): + model_line = next( + line for line in agent_path.read_text().splitlines() if line.startswith("model: ") + ) + models.add(json.loads(model_line.removeprefix("model: "))) + return models + + class TestManagedModelPicker: def test_reads_model_ids_from_managed_picker(self, tmp_path, monkeypatch): path = tmp_path / "managed-settings.json" @@ -235,8 +245,8 @@ def test_restores_model_captured_immediately_before_switch(self, tmp_path, monke def fake_run(argv, **kwargs): captured["argv"] = argv - agents_index = argv.index("--agents") - captured["agents"] = json.loads(argv[agents_index + 1]) + plugin_dir = Path(argv[argv.index("--plugin-dir") + 1]) + captured["plugin_models"] = _plugin_agent_models(plugin_dir) captured["routed_model"] = kwargs["route_prompt"]("fix the parser") generated = Path(argv[argv.index("--settings") + 1]) captured["settings"] = json.loads(generated.read_text()) @@ -274,7 +284,7 @@ def fake_run(argv, **kwargs): display_model="Claude Sonnet 5", rationale="Selected for the parser task.", ) - assert {definition["model"] for definition in captured["agents"].values()} == { + assert captured["plugin_models"] == { "system.ai.claude-opus-4-8", "system.ai.claude-sonnet-5", } @@ -417,7 +427,9 @@ def __init__(self, argv, **_kwargs): settings_path = Path(argv[argv.index("--settings") + 1]) captured["settings_path"] = settings_path captured["settings"] = json.loads(settings_path.read_text()) - captured["agents"] = json.loads(argv[argv.index("--agents") + 1]) + plugin_dir = Path(argv[argv.index("--plugin-dir") + 1]) + captured["plugin_dir"] = plugin_dir + captured["plugin_models"] = _plugin_agent_models(plugin_dir) def wait(self): return 4 @@ -449,11 +461,10 @@ def send_signal(self, _signal): assert "UserPromptSubmit" not in settings["hooks"] assert "route-subagent" in str(settings["hooks"]["PreToolUse"]) assert settings["modelOverrides"] == {"claude-opus-4-8": "system.ai.claude-opus-4-8"} - assert {definition["model"] for definition in captured["agents"].values()} == { - "system.ai.claude-opus-4-8" - } + assert captured["plugin_models"] == {"system.ai.claude-opus-4-8"} assert captured["argv"][3:5] == ["--model", "opus"] assert not captured["settings_path"].exists() + assert not captured["plugin_dir"].exists() # The model-setting guard is a first-prompt concern; user settings stay untouched. assert json.loads(user_settings.read_text()) == {"model": "opus"} @@ -636,29 +647,17 @@ def fake_select(workspace, token, task, route_options, resolve, **kwargs): decision_record = json.loads(decisions_path.read_text()) assert decision_record["requested_model"] == "system.ai.claude-opus-4-8" - def test_merges_caller_agents_with_transient_routed_agents(self): - args = v2._with_routed_claude_agents( - [ - "--agents", - json.dumps( - { - "reviewer": { - "description": "Reviews code", - "prompt": "Review the requested code.", - } - } - ), - "--debug", - ], - ["databricks-claude-opus-4-8"], - ) + def test_writes_routed_agents_as_launch_scoped_plugin(self, tmp_path): + plugin_dir = tmp_path / "routing-plugin" - assert args[0] == "--agents" - definitions = json.loads(args[1]) - assert definitions["reviewer"]["prompt"] == "Review the requested code." - routed = definitions[v2._routed_claude_agent_name("system.ai.claude-opus-4-8")] - assert routed["model"] == "system.ai.claude-opus-4-8" - assert args[2:] == ["--debug"] + v2._write_routed_claude_plugin(plugin_dir, ["databricks-claude-opus-4-8"]) + + manifest = json.loads((plugin_dir / ".claude-plugin" / "plugin.json").read_text()) + assert manifest["name"] == v2.CLAUDE_ROUTING_PLUGIN_NAME + assert _plugin_agent_models(plugin_dir) == {"system.ai.claude-opus-4-8"} + agent = next((plugin_dir / "agents").glob("*.md")).read_text() + expected_name = json.dumps(v2._routed_claude_agent_slug("system.ai.claude-opus-4-8")) + assert f"name: {expected_name}" in agent def test_leaves_non_claude_custom_agent_model_unchanged(self): definitions = v2._routed_claude_agent_definitions(["catalog.schema.gpt-5"]) @@ -677,6 +676,11 @@ def test_maps_gateway_claude_ids_to_known_model_metadata(self): "claude-sonnet-5": "system.ai.claude-sonnet-5", } + def test_routed_agent_uses_plugin_qualified_name(self): + assert v2._routed_claude_agent_name("system.ai.glm-5-3") == ( + "ucode-smart-routing:ucode-route-glm-5-3-982d9f93" + ) + def test_model_switch_lock_serializes_routed_sessions(self, tmp_path, monkeypatch): user_settings = tmp_path / "settings.json" user_settings.write_text(json.dumps({"model": "haiku"})) From dc6242a713e794d7486391fabb4f49a32542cec0 Mon Sep 17 00:00:00 2001 From: Lilly Luo Date: Tue, 22 Sep 2026 03:43:37 +0000 Subject: [PATCH 2/4] Reproduce Claude route agent loss during interactive plugin refresh --- scripts/repro_stale_claude_subagent.md | 77 ++ scripts/repro_stale_claude_subagent.py | 660 +++++++++++++++--- tests/README.md | 6 + tests/integration/README.md | 6 + .../test_ug_smart_routing_hooks.py | 2 +- tests/test_repro_stale_claude_subagent.py | 258 +++++++ 6 files changed, 919 insertions(+), 90 deletions(-) create mode 100644 scripts/repro_stale_claude_subagent.md create mode 100644 tests/test_repro_stale_claude_subagent.py diff --git a/scripts/repro_stale_claude_subagent.md b/scripts/repro_stale_claude_subagent.md new file mode 100644 index 000000000..8d9bb2682 --- /dev/null +++ b/scripts/repro_stale_claude_subagent.md @@ -0,0 +1,77 @@ +# Claude route-agent loss on plugin refresh + +Confirmed on Claude Code **2.1.248**, using the CLI agent registration mechanism +in Unity Gateway **0.1.0+9858a84**. The diagnostic launches real Claude directly, +with a small routing hook and inherited child models. It does not exercise Isaac, +the live smart-router service, or GLM inference. + +## Reproduce + +Run from this checkout with a configured Unity Gateway workspace. The script +reuses the active workspace's Claude authentication helper and model aliases. +It creates a disposable working directory and Claude config directory; machine +managed settings still apply. Each experiment makes real model requests. + +```sh +uv run python scripts/repro_stale_claude_subagent.py \ + --interactive --registration cli \ + --claude /path/to/claude/versions/2.1.248 --log-dir /tmp/route-refresh-cli +``` + +Complete the visible onboarding and trust prompts, then enter these in order: + +1. `Use one Explore subagent to reply with exactly BEFORE_RELOAD. Wait for its result.` +2. After the child finishes: `/reload-plugins --force` +3. After `Reloaded:` appears: `Use one Explore subagent to reply with exactly AFTER_RELOAD. Wait for its result. If it fails, report the tool error and do not retry.` +4. `/exit` + +Expected on 2.1.248: the first routed child completes; the second reports +`Agent type 'ucode-route-glm-5-3-982d9f93' not found`. The script checks actual +tool results, child completion notifications, and routing evidence in the debug +log. Exit 1 means reproduced, 0 means both calls completed across refresh, and +2 means inconclusive. Existing log directories must not be reused for a new run. + +Repeat with `--registration plugin` and a different log directory. Expected: +both routed children complete because plugin refresh reloads their definitions +from disk. This tests the registration strategy used by PR #779; the production +plugin's exact model mappings are covered separately by launcher unit tests. + +Restart existing Claude sessions after upgrading UG to this change: the hook's +new plugin-qualified agent names must match the definitions registered at launch. +Upgrading the hook executable alone does not update an already-running registry. + +## Recorded result + +On 2026-09-22, both experiments ran against the actual 2.1.248 interactive TUI: + +| Registration | Before refresh | After refresh | +| --- | --- | --- | +| CLI `--agents` | Child completed | Exact missing route-agent error | +| Plugin `--plugin-dir` | Child completed | Child completed | + +The CLI run logged a routed completion at `03:37:27.451Z` and +`refreshActivePlugins` at `03:37:39.517Z`, followed by the missing-agent tool +result. The plugin run logged completions at `03:39:01.859Z` and +`03:40:03.140Z`, with refresh at `03:39:50.031Z` between them. + +The failure needs neither nested agents nor parallel calls. Claude's interactive +plugin refresh replaces its registry without retaining CLI-defined agents; +the headless refresh path preserves them. Ten earlier headless parallel-spawn +trials did not reproduce the issue. + +This establishes a causal reproduction, not the trigger in the original user's +session. Check that session's debug logs for `Auto-refreshing plugins` or +`refreshActivePlugins` before the first missing-agent error to connect it to +automatic plugin refresh. + +## Analyze saved evidence without inference + +```sh +uv run python scripts/repro_stale_claude_subagent.py \ + --registration cli --analyze-reload /tmp/route-refresh-cli/cli-01 --exit-code 0 +``` + +Supply the known Claude process exit code, not the diagnostic script's exit +code. For the plugin run use `--registration plugin` and its `plugin-01` +directory. Logs remain local and can contain workspace details; do not publish +raw settings, credentials, or unreviewed debug logs. diff --git a/scripts/repro_stale_claude_subagent.py b/scripts/repro_stale_claude_subagent.py index 0dfe10cb3..9f9cf7c81 100644 --- a/scripts/repro_stale_claude_subagent.py +++ b/scripts/repro_stale_claude_subagent.py @@ -1,118 +1,600 @@ #!/usr/bin/env python3 -"""Replay the parallel Claude agent-registration failure captured in the TUI. +"""Reproduce CLI agent loss on Claude 2.1.248's interactive plugin refresh. -The failing session launched four top-level Agent calls concurrently. Its agent -registry contained built-in and plugin agents but none of the route agents passed -through ``--agents``. This script sends the same four concurrent hook payloads and -checks whether each selected route agent exists in that captured registry shape. +Use --interactive for a before/reload/after experiment with a real Claude TUI. +A PreToolUse hook rewrites Explore to a generated agent. CLI registration loses +that agent on /reload-plugins --force; plugin registration reloads it from disk. +Both modes use inherited child models, not the production GLM model. The default +headless four-call probe does not exercise the TUI refresh path. -Before the fix: - uv run python scripts/repro_stale_claude_subagent.py --registration cli --expect broken - -After the fix: - uv run python scripts/repro_stale_claude_subagent.py --registration plugin --expect fixed +Examples: + uv run python scripts/repro_stale_claude_subagent.py --interactive --registration cli + uv run python scripts/repro_stale_claude_subagent.py --interactive --registration plugin + uv run python scripts/repro_stale_claude_subagent.py --analyze /tmp/run/claude.log --exit-code 0 """ from __future__ import annotations import argparse import json +import os +import shlex +import subprocess +import sys import tempfile -from concurrent.futures import ThreadPoolExecutor +import xml.etree.ElementTree as ET +from dataclasses import dataclass from pathlib import Path -from unittest.mock import patch - -from ucode.smart_routing import routing, v2 - -AVAILABLE_MODEL = "system.ai.glm-5-3" -AVAILABLE_AGENTS = { - "Explore", - "Plan", - "general-purpose", - "model-orchestrator:explorer", - "model-orchestrator:worker", -} -TASKS = ( - "Audit anti-patterns and type safety", - "Audit architecture and complexity", - "Audit test quality", - "Audit hooks, data-fetching, and correctness", -) - - -def _plugin_agents(plugin_dir: Path) -> set[str]: - agents = set() - for agent_path in (plugin_dir / "agents").glob("*.md"): - name_line = next( - line for line in agent_path.read_text().splitlines() if line.startswith("name: ") + +PLUGIN_NAME = "ucode-smart-routing-repro" +ROUTE_AGENT = "ucode-route-glm-5-3-982d9f93" +PROMPT = """In your first assistant response, call the Agent tool exactly four times in +parallel. Use subagent_type Explore for every call. Give each agent one distinct task: +reply with A, reply with B, reply with C, and reply with D. Do not perform the tasks +yourself and do not call any other tool. After all four return, summarize their replies.""" + + +@dataclass(frozen=True) +class Attempt: + outcome: str + tool_calls: int + returncode: int | None + log_path: Path + registration_evidence: str + routed_starts: int + completions: int + reason: str + + +def _agent_definition(index: int) -> dict[str, object]: + return { + "description": f"Registration race reproduction agent {index}", + "prompt": "Reply with exactly the requested letter and nothing else.", + "model": "inherit", + "tools": [], + } + + +def _agent_names(count: int) -> list[str]: + return [ROUTE_AGENT, *(f"ucode-route-repro-{index:03d}" for index in range(count - 1))] + + +def _write_plugin(plugin_dir: Path, count: int) -> None: + manifest_dir = plugin_dir / ".claude-plugin" + agents_dir = plugin_dir / "agents" + manifest_dir.mkdir(parents=True) + agents_dir.mkdir(parents=True) + (manifest_dir / "plugin.json").write_text( + json.dumps( + { + "name": PLUGIN_NAME, + "version": "1.0.0", + "description": "Parallel agent registration race reproduction.", + "author": {"name": "Databricks"}, + } + ) + ) + for index, name in enumerate(_agent_names(count)): + definition = _agent_definition(index) + (agents_dir / f"{name}.md").write_text( + "\n".join( + [ + "---", + f"name: {json.dumps(name)}", + f"description: {json.dumps(definition['description'])}", + "model: inherit", + "tools: []", + "---", + "", + str(definition["prompt"]), + "", + ] + ) ) - name = json.loads(name_line.removeprefix("name: ")) - agents.add(f"{v2.CLAUDE_ROUTING_PLUGIN_NAME}:{name}") - return agents - - -def _route(task: str) -> str: - output = v2.route_claude_pre_tool_use( - { - "tool_name": "Agent", - "tool_input": { - "subagent_type": "Explore", - "prompt": task, - "model": "sonnet", - }, - }, - workspace="https://example.databricks.com", - token="test-token", - available_models=[AVAILABLE_MODEL], + + +def _write_hook(hook_path: Path) -> None: + hook_path.write_text( + """#!/usr/bin/env python3 +import json +import sys + +payload = json.load(sys.stdin) +updated = dict(payload["tool_input"]) +updated.pop("model", None) +updated["subagent_type"] = sys.argv[1] +print(json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "allow", + "updatedInput": updated, + } +})) +""" + ) + + +def _write_settings( + settings_path: Path, + hook_path: Path, + routed_name: str, + base_settings: dict[str, object], +) -> None: + settings = dict(base_settings) + settings["hooks"] = { + "PreToolUse": [ + { + "matcher": "Agent|Task", + "hooks": [ + { + "type": "command", + "command": shlex.join([sys.executable, str(hook_path), routed_name]), + "timeout": 10, + } + ], + } + ] + } + settings_path.write_text(json.dumps(settings)) + + +def _ucode_claude_settings(state_path: Path) -> dict[str, object]: + if not state_path.is_file(): + return {} + state = json.loads(state_path.read_text()) + current_workspace = state.get("current_workspace") + workspaces = state.get("workspaces") + if not isinstance(current_workspace, str) or not isinstance(workspaces, dict): + return {} + workspace = workspaces.get(current_workspace) + if not isinstance(workspace, dict): + return {} + agents = workspace.get("agents") + claude = agents.get("claude") if isinstance(agents, dict) else None + if not isinstance(claude, dict): + return {} + settings: dict[str, object] = {} + auth_command = claude.get("auth_command") + if isinstance(auth_command, str): + settings["apiKeyHelper"] = auth_command + env = claude.get("env") + if isinstance(env, dict): + configured_env = dict(env) + configured_env.setdefault( + "ANTHROPIC_CUSTOM_HEADERS", "x-databricks-use-coding-agent-mode: true" + ) + claude_models = workspace.get("claude_models") + if isinstance(claude_models, dict): + for family in ("opus", "sonnet", "haiku"): + model = claude_models.get(family) + if isinstance(model, str): + configured_env[f"ANTHROPIC_DEFAULT_{family.upper()}_MODEL"] = model + settings["modelOverrides"] = { + model.removeprefix("system.ai."): model + for model in claude_models.values() + if isinstance(model, str) and model.startswith("system.ai.claude-") + } + settings["env"] = configured_env + return settings + + +def _message_blocks(event: dict) -> list[dict]: + content = event.get("message", {}).get("content", []) + return ( + [block for block in content if isinstance(block, dict)] if isinstance(content, list) else [] + ) + + +def _classify( + output: str, returncode: int | None, log_path: Path, routed_name: str = ROUTE_AGENT +) -> Attempt: + events = [] + for line in output.splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(event, dict): + events.append(event) + + init = next( + ( + event + for event in events + if event.get("type") == "system" and event.get("subtype") == "init" + ), + None, ) - return output["hookSpecificOutput"]["updatedInput"]["subagent_type"] - - -def reproduce(registration: str) -> list[str]: - registered_agents = set(AVAILABLE_AGENTS) - with tempfile.TemporaryDirectory() as temp_dir: - if registration == "plugin": - plugin_dir = Path(temp_dir) / "routing-plugin" - v2._write_routed_claude_plugin(plugin_dir, [AVAILABLE_MODEL]) - registered_agents.update(_plugin_agents(plugin_dir)) - - decision = routing.RoutingDecision(model=AVAILABLE_MODEL, raw_model="glm-5-3") - with ( - patch( - "ucode.smart_routing.v2.routing.select_route", - return_value=(decision, None), - ), - ThreadPoolExecutor(max_workers=len(TASKS)) as executor, + registration_evidence = "unknown" + if init is not None and isinstance(init.get("agents"), list): + registration_evidence = ( + "present_at_first_init" if routed_name in init["agents"] else "absent_at_first_init" + ) + assistants = [ + event + for event in events + if event.get("type") == "assistant" and event.get("parent_tool_use_id") is None + ] + first_message = assistants[0].get("message", {}).get("id") if assistants else None + first_calls = {} + all_calls = {} + for event in assistants: + for block in _message_blocks(event): + if block.get("type") == "tool_use" and block.get("name") in {"Agent", "Task"}: + all_calls[block["id"]] = block + if first_message and event.get("message", {}).get("id") == first_message: + first_calls[block["id"]] = block + + errors = [] + for event in events: + if event.get("type") == "user" and event.get("parent_tool_use_id") is None: + for block in _message_blocks(event): + if ( + block.get("type") == "tool_result" + and block.get("is_error") + and block.get("tool_use_id") in all_calls + ): + errors.append(json.dumps(block.get("content", ""))) + if ( + event.get("type") == "system" + and event.get("subtype") == "task_notification" + and event.get("tool_use_id") in all_calls + and event.get("status") == "failed" ): - selected_agents = list(executor.map(_route, TASKS)) + errors.append(str(event.get("summary", ""))) + + starts = { + event["tool_use_id"] + for event in events + if event.get("type") == "system" + and event.get("subtype") == "task_started" + and event.get("tool_use_id") in first_calls + and event.get("subagent_type") == routed_name + and event.get("spawn_depth") == 1 + } + completions = { + event["tool_use_id"] + for event in events + if event.get("type") == "system" + and event.get("subtype") == "task_notification" + and event.get("tool_use_id") in starts + and event.get("status") == "completed" + } + final_results = [event for event in events if event.get("type") == "result"] + outcome = "inconclusive" + reason = "Require four Explore calls in the first message, routed starts, and completions." + if any(f"Agent type '{routed_name}' not found" in error for error in errors): + outcome = "registry_failure" + reason = "Claude reported the missing route agent in an actual tool failure." + elif errors: + reason = "Agent tool failed for another reason; inspect the log." + elif ( + returncode == 0 + and registration_evidence == "present_at_first_init" + and len(first_calls) == len(all_calls) == len(starts) == len(completions) == 4 + and all( + call.get("input", {}).get("subagent_type") == "Explore" for call in first_calls.values() + ) + and final_results + and all( + event.get("subtype") == "success" and event.get("is_error") is False + for event in final_results + ) + ): + outcome = "no_failure_observed" + reason = "All four routed children completed; this does not rule out a race." + return Attempt( + outcome, + len(all_calls), + returncode, + log_path, + registration_evidence, + len(starts), + len(completions), + reason, + ) + +def _classify_reload(attempt_dir: Path, routed_name: str, returncode: int | None) -> Attempt: + transcripts = list((attempt_dir / "claude-config/projects").glob("*/*.jsonl")) + debug_path = attempt_dir / "debug.log" + debug = debug_path.read_text() if debug_path.is_file() else "" + phase = "before" + markers = {"before": "BEFORE_RELOAD", "after": "AFTER_RELOAD"} + calls = {} + launched = {} + completed = set() + missing = set() + reload_requested = False + if len(transcripts) == 1: + for line in transcripts[0].read_text().splitlines(): + event = json.loads(line) + if event.get("isSidechain"): + continue + content = event.get("message", {}).get("content", "") + if event.get("type") == "user" and isinstance(content, str): + if "/reload-plugins" in content: + reload_requested = True + if content.startswith(""): + try: + notification = ET.fromstring(content) + except ET.ParseError: + continue + tool_id = notification.findtext("tool-use-id") + agent_id = launched.get(tool_id) + if ( + agent_id + and calls.get(tool_id) == phase + and notification.findtext("status") == "completed" + and notification.findtext("result", "").strip() == markers[phase] + and f"agentId={agent_id} agentType={routed_name} exitPath=completed" + in debug + ): + completed.add(tool_id) + if ( + reload_requested + and event.get("type") == "system" + and event.get("subtype") == "local_command" + and str(event.get("content", "")).startswith("Reloaded:") + and "refreshActivePlugins:" in debug + ): + phase = "after" + for block in _message_blocks(event): + if ( + event.get("type") == "assistant" + and block.get("type") == "tool_use" + and block.get("name") in {"Agent", "Task"} + ): + inputs = block.get("input", {}) + calls[block["id"]] = ( + phase + if inputs.get("subagent_type") == "Explore" + and markers[phase] in inputs.get("prompt", "") + else "unexpected" + ) + if event.get("type") == "user" and block.get("type") == "tool_result": + tool_id = block.get("tool_use_id") + result = event.get("toolUseResult") + if isinstance(result, dict) and result.get("agentId"): + launched[tool_id] = result["agentId"] + if block.get("is_error") and f"Agent type '{routed_name}' not found" in str( + block.get("content") + ): + missing.add(tool_id) + before = {tool_id for tool_id, call_phase in calls.items() if call_phase == "before"} + after = {tool_id for tool_id, call_phase in calls.items() if call_phase == "after"} + outcome = "inconclusive" + reason = "Require a routed completion before reload, reload completion, and one call after it." + if len(calls) == 2 and len(before) == len(after) == 1 and before <= completed: + if after <= missing: + outcome = "registry_failure" + reason = "Routed child completed before plugin refresh; the same route was missing afterward." + elif after <= completed and returncode == 0: + outcome = "no_failure_observed" + reason = ( + "Routed children completed both before and after the interactive plugin refresh." + ) + return Attempt( + outcome, + len(calls), + returncode, + debug_path, + "proved_before_refresh" if before and before <= completed else "unknown", + len(launched), + len(completed), + reason, + ) + + +def run_attempt( + claude: str, + registration: str, + agent_count: int, + model: str, + timeout: int, + log_dir: Path, + attempt_number: int, + base_settings: dict[str, object], + interactive: bool = False, +) -> Attempt: + attempt_dir = log_dir / f"{registration}-{attempt_number:02d}" + attempt_dir.mkdir(parents=True) + hook_path = attempt_dir / "route_hook.py" + settings_path = attempt_dir / "settings.json" + _write_hook(hook_path) + + routed_name = ROUTE_AGENT + registration_args: list[str] if registration == "cli": - prefix = f"{v2.CLAUDE_ROUTING_PLUGIN_NAME}:" - selected_agents = [agent.removeprefix(prefix) for agent in selected_agents] - return [agent for agent in selected_agents if agent not in registered_agents] + definitions = { + name: _agent_definition(index) for index, name in enumerate(_agent_names(agent_count)) + } + registration_args = ["--agents", json.dumps(definitions, separators=(",", ":"))] + else: + plugin_dir = attempt_dir / "plugin" + _write_plugin(plugin_dir, agent_count) + routed_name = f"{PLUGIN_NAME}:{ROUTE_AGENT}" + registration_args = ["--plugin-dir", str(plugin_dir)] + _write_settings(settings_path, hook_path, routed_name, base_settings) + + if interactive: + config_dir = attempt_dir / "claude-config" + config_dir.mkdir() + command = [ + claude, + "--model", + model, + "--settings", + str(settings_path), + "--setting-sources", + "", + "--debug-file", + str(attempt_dir / "debug.log"), + "--tools", + "Agent", + "--allowed-tools", + "Agent", + *registration_args, + ] + print( + "Interactive plugin-refresh experiment (real model requests):\n" + "1. Finish visible onboarding/trust prompts for this disposable directory.\n" + "2. Ask: Use one Explore subagent to reply with exactly BEFORE_RELOAD.\n" + "3. Wait for the subagent to finish, then enter: /reload-plugins --force\n" + "4. Wait for Reloaded, then ask: Use one Explore subagent to reply with exactly AFTER_RELOAD.\n" + "5. Record the actual tool result and exit with /exit.\n" + f"Registration: {registration}; agent: {routed_name}\n" + f"Evidence directory: {attempt_dir}\n" + "This tests plugin refresh; it does not establish what triggered the user's incident.", + flush=True, + ) + try: + completed = subprocess.run( + command, + cwd=attempt_dir, + env={ + **os.environ, + "CLAUDE_CONFIG_DIR": str(config_dir), + "DISABLE_AUTOUPDATER": "1", + }, + timeout=timeout, + check=False, + ) + returncode = completed.returncode + except subprocess.TimeoutExpired: + returncode = 124 + return _classify_reload(attempt_dir, routed_name, returncode) + + command = [ + claude, + "--print", + "--verbose", + "--output-format", + "stream-json", + "--max-turns", + "3", + "--model", + model, + "--settings", + str(settings_path), + "--allowed-tools", + "Agent", + *registration_args, + PROMPT, + ] + try: + completed = subprocess.run( + command, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + output = completed.stdout + returncode = completed.returncode + except subprocess.TimeoutExpired as exc: + captured = exc.stdout or "" + output = captured.decode(errors="replace") if isinstance(captured, bytes) else captured + output += "\nTIMED OUT" + returncode = 124 + log_path = attempt_dir / "claude.log" + log_path.write_text(output) + return _classify(output, returncode, log_path, routed_name) def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--registration", choices=("cli", "plugin"), default="plugin") - parser.add_argument("--expect", choices=("broken", "fixed"), default="fixed") + parser.add_argument("--registration", choices=("cli", "plugin"), default="cli") + parser.add_argument( + "--interactive", + action="store_true", + help="Open an isolated real TUI for the before/reload/after experiment (one attempt)", + ) + parser.add_argument( + "--analyze", + type=Path, + nargs="+", + help="Inspect existing JSONL logs without launching Claude", + ) + parser.add_argument( + "--analyze-reload", + type=Path, + nargs="+", + help="Inspect existing interactive attempt directories without launching Claude", + ) + parser.add_argument( + "--exit-code", type=int, help="Known process exit code for --analyze; otherwise unknown" + ) + parser.add_argument("--attempts", type=int, default=1) + parser.add_argument("--agent-count", type=int, default=1) + parser.add_argument("--model", default="haiku") + parser.add_argument("--claude", default="claude") + parser.add_argument("--timeout", type=int, default=480) + parser.add_argument("--log-dir", type=Path) + parser.add_argument("--ucode-state", type=Path, default=Path.home() / ".ucode/state.json") args = parser.parse_args() + if args.attempts < 1 or args.agent_count < 1 or args.timeout < 1: + parser.error("--attempts, --agent-count, and --timeout must be positive") + if args.interactive and (args.attempts != 1 or args.analyze or args.analyze_reload): + parser.error("--interactive requires one attempt and cannot be combined with analysis") + if args.analyze and args.analyze_reload: + parser.error("Choose --analyze or --analyze-reload") + if args.analyze_reload: + routed_name = ROUTE_AGENT if args.registration == "cli" else f"{PLUGIN_NAME}:{ROUTE_AGENT}" + return _report( + [_classify_reload(path, routed_name, args.exit_code) for path in args.analyze_reload] + ) - missing_agents = reproduce(args.registration) - if missing_agents: + if args.analyze: + routed_name = ROUTE_AGENT if args.registration == "cli" else f"{PLUGIN_NAME}:{ROUTE_AGENT}" + attempts = [ + _classify(path.read_text(), args.exit_code, path, routed_name) for path in args.analyze + ] + return _report(attempts) + + log_dir = args.log_dir or Path(tempfile.mkdtemp(prefix="claude-agent-race-")) + log_dir.mkdir(parents=True, exist_ok=True) + base_settings = _ucode_claude_settings(args.ucode_state) + attempts = [] + for attempt_number in range(1, args.attempts + 1): + attempt = run_attempt( + args.claude, + args.registration, + args.agent_count, + args.model, + args.timeout, + log_dir, + attempt_number, + base_settings, + args.interactive, + ) + attempts.append(attempt) print( - f"Agent type '{missing_agents[0]}' not found. " - f"Available agents: {', '.join(sorted(AVAILABLE_AGENTS))}" + f"{attempt_number}/{args.attempts}: {attempt.outcome} " + f"(Agent calls={attempt.tool_calls}, exit={attempt.returncode})", + flush=True, + ) + return _report(attempts) + + +def _report(attempts: list[Attempt]) -> int: + for attempt in attempts: + print( + f"{attempt.log_path}: {attempt.outcome}; {attempt.registration_evidence}; " + f"routed starts={attempt.routed_starts}; completions={attempt.completions}. " + f"{attempt.reason}", + flush=True, ) - print(f"FAILED: {len(missing_agents)} concurrent Agent calls selected missing agents") - else: - print(f"PASS: all {len(TASKS)} concurrent Agent calls selected registered plugin agents") - actual = "broken" if missing_agents else "fixed" - if actual != args.expect: - print(f"Expected {args.expect}, observed {actual}") + counts = { + outcome: sum(attempt.outcome == outcome for attempt in attempts) + for outcome in ("registry_failure", "no_failure_observed", "inconclusive") + } + print(f"Results: {counts}", flush=True) + if counts["registry_failure"]: return 1 - return 0 + return 2 if counts["inconclusive"] else 0 if __name__ == "__main__": diff --git a/tests/README.md b/tests/README.md index 5c671fdd3..e96f666c0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -25,6 +25,12 @@ the selected and empty checkboxes. These are local component checks, not live ga ## CUJ coverage matrix +`test_repro_stale_claude_subagent.py` checks the diagnostic script's evidence +classification using synthetic event fixtures. These are parser tests, not a +reproduction of Claude's missing route-agent incident. The script's +`--interactive` mode exercises Claude's plugin refresh in a real TUI; its +headless mode does not. Neither mode exercises Isaac's launcher. + These are **implemented assertions**, not a claim that every version passes. Consult the run's JUnit report and artifacts for results. Each function states its **Scenario** and **Expected** outcome and shows its configure and launch diff --git a/tests/integration/README.md b/tests/integration/README.md index 8d4fc73e2..6b4997103 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -143,6 +143,12 @@ arm. The agent's interactive spawn decision, interactive explicit-model bypass, dedicated smart-routing CI shards remain deferred; unit/component routing tests do not establish that live behavior. +The Claude hook contract now expects the plugin-qualified +`ucode-smart-routing:ucode-route-*` agent name. Plugin-refresh survival is tested +separately by the manual interactive diagnostic +`scripts/repro_stale_claude_subagent.py --interactive`; it is not claimed as +automated integration coverage here. + The relayed CUJ launches Claude through a relayed (subscription-relay) MPS and completes a file task on two models: a bare Anthropic id the subscription serves directly (`route=relay`) and a Databricks-hosted `system.ai` id the loopback proxy diff --git a/tests/integration/test_ug_smart_routing_hooks.py b/tests/integration/test_ug_smart_routing_hooks.py index 255aafb34..e120308f9 100644 --- a/tests/integration/test_ug_smart_routing_hooks.py +++ b/tests/integration/test_ug_smart_routing_hooks.py @@ -86,7 +86,7 @@ def test_smart_routing_claude_route_subagent_hook(live_session, workspace): assert SMART_ROUTING_SUBAGENT_NOTICE in output["systemMessage"], output updated = hook["updatedInput"] assert "model" not in updated, updated - assert updated["subagent_type"].startswith("ucode-route-"), updated + assert updated["subagent_type"].startswith("ucode-smart-routing:ucode-route-"), updated assert updated["prompt"] == payload["tool_input"]["prompt"], updated assert updated["description"] == payload["tool_input"]["description"], updated diff --git a/tests/test_repro_stale_claude_subagent.py b/tests/test_repro_stale_claude_subagent.py new file mode 100644 index 000000000..9ec509d43 --- /dev/null +++ b/tests/test_repro_stale_claude_subagent.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts import repro_stale_claude_subagent as repro + + +def completed_events(): + events = [{"type": "system", "subtype": "init", "agents": [repro.ROUTE_AGENT]}] + for index in range(4): + tool_id = f"call-{index}" + events.extend( + [ + { + "type": "assistant", + "message": { + "id": "first-message", + "content": [ + { + "type": "tool_use", + "name": "Agent", + "id": tool_id, + "input": {"subagent_type": "Explore"}, + } + ], + }, + }, + { + "type": "system", + "subtype": "task_started", + "tool_use_id": tool_id, + "subagent_type": repro.ROUTE_AGENT, + "spawn_depth": 1, + }, + { + "type": "system", + "subtype": "task_notification", + "tool_use_id": tool_id, + "status": "completed", + }, + ] + ) + events.append({"type": "result", "subtype": "success", "is_error": False}) + return events + + +def classify(events, returncode=0): + return repro._classify( + "\n".join(json.dumps(event) for event in events), returncode, Path("claude.log") + ) + + +def test_classifies_completed_routed_children_without_claiming_a_fix(): + result = classify(completed_events()) + assert result.outcome == "no_failure_observed" + assert result.registration_evidence == "present_at_first_init" + assert result.routed_starts == result.completions == 4 + + +@pytest.mark.parametrize("missing_subtype", ["task_started", "task_notification", "init"]) +def test_four_tool_calls_alone_are_insufficient(missing_subtype): + events = [event for event in completed_events() if event.get("subtype") != missing_subtype] + assert classify(events).outcome == "inconclusive" + + +def test_sequential_batches_are_not_parallel_first_use(): + events = completed_events() + for index, event in enumerate(events): + if event["type"] == "assistant": + event["message"]["id"] = f"message-{index}" + assert classify(events).outcome == "inconclusive" + + +def test_nested_calls_do_not_count_as_top_level(): + events = completed_events() + for event in events: + if event["type"] == "assistant": + event["parent_tool_use_id"] = "parent" + assert classify(events).outcome == "inconclusive" + + +def test_unrouted_explore_completion_is_not_a_success(): + events = completed_events() + for event in events: + if event.get("subtype") == "task_started": + event["subagent_type"] = "Explore" + assert classify(events).outcome == "inconclusive" + + +def test_assistant_claim_of_missing_agent_is_not_registry_failure(): + events = completed_events() + events.append( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": f"Agent type '{repro.ROUTE_AGENT}' not found", + } + ] + }, + } + ) + assert classify(events).outcome == "no_failure_observed" + + +@pytest.mark.parametrize("registered", [True, False]) +def test_actual_tool_error_records_initial_registration(registered): + events = completed_events() + events[0]["agents"] = [repro.ROUTE_AGENT] if registered else [] + events.append( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "call-0", + "is_error": True, + "content": f"Agent type '{repro.ROUTE_AGENT}' not found. Available agents: Explore", + } + ] + }, + } + ) + result = classify(events) + assert result.outcome == "registry_failure" + assert result.registration_evidence == ( + "present_at_first_init" if registered else "absent_at_first_init" + ) + + +@pytest.mark.parametrize("returncode", [None, 1, 124]) +def test_unknown_or_failed_exit_is_inconclusive(returncode): + assert classify(completed_events(), returncode).outcome == "inconclusive" + + +def test_failed_completion_is_not_success(): + events = completed_events() + events[3]["status"] = "failed" + events[3]["summary"] = "Authentication failed" + assert classify(events).outcome == "inconclusive" + + +def reload_evidence(tmp_path, *, missing_after=False, complete_before=True, reload=True): + events = [] + debug = [] + for phase in ("before", "after"): + if phase == "after" and reload: + events.extend( + [ + { + "type": "user", + "message": {"content": "/reload-plugins"}, + }, + { + "type": "system", + "subtype": "local_command", + "content": "Reloaded: 6 agents", + }, + ] + ) + debug.append("refreshActivePlugins: 6 agents") + marker = f"{phase.upper()}_RELOAD" + events.append( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": "Agent", + "id": phase, + "input": {"subagent_type": "Explore", "prompt": f"Reply with {marker}"}, + } + ] + }, + } + ) + if phase == "after" and missing_after: + events.append( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": phase, + "is_error": True, + "content": f"Agent type '{repro.ROUTE_AGENT}' not found", + } + ] + }, + } + ) + continue + events.append( + { + "type": "user", + "toolUseResult": {"agentId": f"child-{phase}"}, + "message": {"content": [{"type": "tool_result", "tool_use_id": phase}]}, + } + ) + if phase == "before" and not complete_before: + continue + debug.append(f"agentId=child-{phase} agentType={repro.ROUTE_AGENT} exitPath=completed") + events.append( + { + "type": "user", + "message": { + "content": ( + f"{phase}" + f"completed{marker}" + ) + }, + } + ) + projects = tmp_path / "claude-config/projects/project" + projects.mkdir(parents=True) + (projects / "session.jsonl").write_text("\n".join(json.dumps(event) for event in events)) + (tmp_path / "debug.log").write_text("\n".join(debug)) + + +def test_reload_requires_success_before_actual_missing_agent_error(tmp_path): + reload_evidence(tmp_path, missing_after=True) + result = repro._classify_reload(tmp_path, repro.ROUTE_AGENT, 0) + assert result.outcome == "registry_failure" + assert result.registration_evidence == "proved_before_refresh" + assert result.completions == 1 + + +def test_reload_survival_requires_two_routed_completions(tmp_path): + reload_evidence(tmp_path) + result = repro._classify_reload(tmp_path, repro.ROUTE_AGENT, 0) + assert result.outcome == "no_failure_observed" + assert result.completions == 2 + + +@pytest.mark.parametrize("missing_after", [False, True]) +def test_missing_before_completion_cannot_prove_registration_loss(tmp_path, missing_after): + reload_evidence(tmp_path, missing_after=missing_after, complete_before=False) + assert repro._classify_reload(tmp_path, repro.ROUTE_AGENT, 0).outcome == "inconclusive" + + +def test_no_reload_cannot_prove_refresh_survival(tmp_path): + reload_evidence(tmp_path, reload=False) + assert repro._classify_reload(tmp_path, repro.ROUTE_AGENT, 0).outcome == "inconclusive" + + +def test_builtin_children_cannot_prove_routed_agent_survival(tmp_path): + reload_evidence(tmp_path) + debug_path = tmp_path / "debug.log" + debug_path.write_text(debug_path.read_text().replace(repro.ROUTE_AGENT, "Explore")) + assert repro._classify_reload(tmp_path, repro.ROUTE_AGENT, 0).outcome == "inconclusive" From c1ab0c3ba19c54458d3cae096f79e903637a0718 Mon Sep 17 00:00:00 2001 From: Lilly Luo Date: Tue, 22 Sep 2026 03:53:35 +0000 Subject: [PATCH 3/4] Add opt-in Claude debug log directory for UG launches --- README.md | 28 ++++++++++++ src/ucode/agents/claude.py | 30 +++++++++++- tests/README.md | 6 +++ tests/test_agent_claude.py | 93 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9ae0bb692..96239e1f6 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,34 @@ Databricks AI Tools are installed only by `ug configure`, never by agent launch commands. Use `--enable-databricks-ai-tools` or `--disable-databricks-ai-tools` with `ug configure` to control installation. +## Claude Debug Logs + +To capture Claude's native debug logs, including plugin refresh and agent-routing +events, set a directory for the next launch: + +```bash +UG_CLAUDE_DEBUG_LOG_DIR="$HOME/ug-debug" isaac +# Or launch directly: +UG_CLAUDE_DEBUG_LOG_DIR="$HOME/ug-debug" ug claude +``` + +This requires a UG version containing this option. Each launch creates a unique +`claude-*.log` file, prints its absolute path to stderr, and retains the file after +exit. Files are created with owner-only permissions. The option applies to normal, +smart-routed, and relayed Claude launches. An explicit `--debug-file` takes +precedence. Unset the variable to stop creating logs; remove retained files when +finished investigating. + +To locate evidence of the missing-agent failure: + +```bash +rg -n 'Auto-refreshing plugins|refreshActivePlugins|Agent type .*not found' "$HOME/ug-debug"/claude-*.log +``` + +This is local capture, not automatic upload or recovery of previous sessions. +Review logs before sharing: Claude debug output can contain prompts, paths, and +other sensitive session details. + ## Managed Files `ug` backs up files before overwriting them. `ug revert` restores backups. diff --git a/src/ucode/agents/claude.py b/src/ucode/agents/claude.py index ee7cc4e67..c62380d27 100644 --- a/src/ucode/agents/claude.py +++ b/src/ucode/agents/claude.py @@ -9,6 +9,7 @@ import signal import socket import subprocess +import tempfile import threading from collections.abc import Callable from pathlib import Path @@ -75,11 +76,12 @@ from ucode.smart_routing.routing import configured_router_name from ucode.state import MANAGED_OVERLAY_KEY, is_tool_managed, mark_tool_managed, save_state from ucode.telemetry import agent_version, ug_version -from ucode.ui import print_note, print_success, print_warning +from ucode.ui import err_console, print_note, print_success, print_warning from .args import LaunchOptions, has_explicit_model_arg GATEWAY_MODEL_DISCOVERY_ENV_VAR = "ENABLE_CLAUDE_CODE_GATEWAY_MODEL_DISCOVERY" +DEBUG_LOG_DIR_ENV_VAR = "UG_CLAUDE_DEBUG_LOG_DIR" # If set, Claude Code launches in headless mode instead of the interactive login flow. CLAUDE_CODE_OAUTH_TOKEN_ENV_VAR = "CLAUDE_CODE_OAUTH_TOKEN" CLAUDE_CONFIG_DIR = Path.home() / ".claude" @@ -1492,12 +1494,38 @@ def token_provider(force_refresh: bool) -> str: raise SystemExit(returncode) +def _with_debug_log(tool_args: list[str]) -> list[str]: + directory = os.environ.get(DEBUG_LOG_DIR_ENV_VAR, "").strip() + if not directory: + return tool_args + for arg in tool_args: + if arg == "--": + break + if arg == "--debug-file" or arg.startswith("--debug-file="): + return tool_args + try: + log_dir = Path(directory).expanduser().resolve() + log_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor, filename = tempfile.mkstemp( + prefix=f"claude-{os.getpid()}-", suffix=".log", dir=log_dir + ) + os.close(descriptor) + except OSError as exc: + raise RuntimeError( + f"Cannot create Claude debug log in {directory!r}: {exc}. " + f"Set {DEBUG_LOG_DIR_ENV_VAR} to a writable directory or unset it." + ) from exc + err_console.print(f"Claude debug log: {filename}", style="dim", markup=False) + return ["--debug-file", filename, *tool_args] + + def launch( state: dict, tool_args: list[str], *, options: LaunchOptions, ) -> None: + tool_args = _with_debug_log(tool_args) binary = SPEC["binary"] workspace = state.get("workspace") if workspace and os.environ.get(GATEWAY_MODEL_DISCOVERY_ENV_VAR) == "1": diff --git a/tests/README.md b/tests/README.md index e96f666c0..7695cd135 100644 --- a/tests/README.md +++ b/tests/README.md @@ -23,6 +23,12 @@ keyboard selection: nothing is selected by default, selecting Codex installs onl Codex, and submitting an empty selection installs nothing. Rendering checks cover the selected and empty checkboxes. These are local component checks, not live gateway tests. +`TestClaudeDebugLogs` in `test_agent_claude.py` covers opt-in native debug-log +capture through `UG_CLAUDE_DEBUG_LOG_DIR`: unique private files, caller flag +precedence, stderr-only notices, actionable filesystem errors, and forwarding +through normal, smart-routed, and relayed launches. These are component checks; +they do not claim remote log retrieval or a live Claude session. + ## CUJ coverage matrix `test_repro_stale_claude_subagent.py` checks the diagnostic script's evidence diff --git a/tests/test_agent_claude.py b/tests/test_agent_claude.py index 0b3fdb3b7..3b2b5e20a 100644 --- a/tests/test_agent_claude.py +++ b/tests/test_agent_claude.py @@ -1701,6 +1701,99 @@ def boom(name, entry, scope=claude.MCP_USER_SCOPE): assert result["workspace"] == WS +class TestClaudeDebugLogs: + @pytest.fixture(autouse=True) + def clear_debug_env(self, monkeypatch): + monkeypatch.delenv(claude.DEBUG_LOG_DIR_ENV_VAR, raising=False) + + def test_disabled_leaves_arguments_and_files_unchanged(self, tmp_path, capsys): + arguments = ["--print", "hello"] + existing = set(tmp_path.iterdir()) + assert claude._with_debug_log(arguments) == arguments + assert set(tmp_path.iterdir()) == existing + assert capsys.readouterr().err == "" + + def test_unique_private_logs_and_stderr_notice(self, tmp_path, monkeypatch, capsys): + directory = tmp_path / "logs with spaces" + monkeypatch.setenv(claude.DEBUG_LOG_DIR_ENV_VAR, str(directory)) + arguments = ["--print", "hello"] + + first = claude._with_debug_log(arguments) + second = claude._with_debug_log(arguments) + + assert arguments == ["--print", "hello"] + assert first[0] == second[0] == "--debug-file" + assert first[2:] == second[2:] == arguments + assert first[1] != second[1] + for filename in (first[1], second[1]): + log = Path(filename) + assert log.is_absolute() and log.parent == directory + assert log.is_file() + if os.name != "nt": + assert log.stat().st_mode & 0o777 == 0o600 + output = capsys.readouterr() + assert output.out == "" + assert "Claude debug log:" in output.err + + @pytest.mark.parametrize( + "arguments", [["--debug-file", "chosen.log"], ["--debug-file=chosen.log"]] + ) + def test_explicit_debug_file_wins(self, tmp_path, monkeypatch, arguments): + directory = tmp_path / "unused" + monkeypatch.setenv(claude.DEBUG_LOG_DIR_ENV_VAR, str(directory)) + assert claude._with_debug_log(arguments) == arguments + assert not directory.exists() + + def test_prompt_separator_does_not_disable_logging(self, tmp_path, monkeypatch): + monkeypatch.setenv(claude.DEBUG_LOG_DIR_ENV_VAR, str(tmp_path)) + arguments = ["--", "--debug-file=prompt-text"] + result = claude._with_debug_log(arguments) + assert result[0] == "--debug-file" + assert result[2:] == arguments + + def test_expands_home_directory(self, tmp_path, monkeypatch): + monkeypatch.setattr(claude.Path, "home", lambda: tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv(claude.DEBUG_LOG_DIR_ENV_VAR, "~/logs") + result = claude._with_debug_log([]) + assert Path(result[1]).parent == tmp_path / "logs" + + def test_invalid_directory_is_actionable(self, tmp_path, monkeypatch): + invalid = tmp_path / "file" + invalid.write_text("keep me") + monkeypatch.setenv(claude.DEBUG_LOG_DIR_ENV_VAR, str(invalid)) + with pytest.raises(RuntimeError, match="UG_CLAUDE_DEBUG_LOG_DIR.*writable directory"): + claude._with_debug_log([]) + assert invalid.read_text() == "keep me" + + @pytest.mark.parametrize("mode", ["normal", "smart", "relayed", "pinned"]) + def test_log_forwarded_through_each_launch_path(self, tmp_path, monkeypatch, mode): + monkeypatch.setenv(claude.DEBUG_LOG_DIR_ENV_VAR, str(tmp_path)) + captured = [] + monkeypatch.setattr(claude, "exec_or_spawn", lambda argv: captured.append(argv)) + monkeypatch.setattr( + claude, "_launch_relayed", lambda state, binary, args: captured.append(args) + ) + monkeypatch.setattr( + claude.smart_routing_v2, + "launch_claude", + lambda state, args, **kwargs: captured.append(args), + ) + state = {"claude_relayed": mode == "relayed"} + options = LaunchOptions( + launch_smart_routing=mode == "smart", + user_pinned_model="sonnet" if mode == "pinned" else None, + ) + claude.launch(state, ["--print", "hello"], options=options) + + assert len(captured) == 1 + argv = captured[0] + assert argv[-2:] == ["--print", "hello"] + logfile = Path(argv[argv.index("--debug-file") + 1]) + assert logfile.is_file() + assert logfile.parent == tmp_path + + class TestClaudeLaunch: def test_gateway_discovery_enabled_for_relayed_provider(self, monkeypatch): calls: list[tuple[dict, str, list[str]]] = [] From 19534176e46e222ef8c2e7e03eb9b49bb80a0c8a Mon Sep 17 00:00:00 2001 From: Lilly Luo Date: Tue, 22 Sep 2026 04:02:09 +0000 Subject: [PATCH 4/4] Clean abandoned Claude routing plugins and bound plugin lifetime --- README.md | 14 +++ src/ucode/agents/claude.py | 1 + src/ucode/smart_routing/claude_pty.py | 3 + src/ucode/smart_routing/v2.py | 130 +++++++++++++++------ tests/README.md | 6 + tests/conftest.py | 2 + tests/test_claude_smart_routing_v2.py | 159 +++++++++++++++++++++++++- 7 files changed, 276 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 96239e1f6..08a1bdd1c 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,20 @@ Databricks AI Tools are installed only by `ug configure`, never by agent launch commands. Use `--enable-databricks-ai-tools` or `--disable-databricks-ai-tools` with `ug configure` to control installation. +## Claude Routing Plugin Lifetime + +Smart routing's generated agent definitions are a temporary, per-launch plugin, +not a globally installed Claude plugin. A launch with smart routing disabled +passes no generated plugin to Claude. Either `ENABLE_SMART_ROUTING_V2=1` or +`ENABLE_SMART_ROUTING_SUBAGENT_ONLY=1` enables routing (workspace-managed routing +configuration can also enable it). + +The generated plugin is removed on exit, including setup and process-launch +failures. Every subsequent Claude launch also removes abandoned plugin directories +left by terminated UG processes. A lock inherited by Claude protects plugins +still used by active sessions, even if their UG parent has exited. Cleanup leaves +user plugins and unrelated directories alone. + ## Claude Debug Logs To capture Claude's native debug logs, including plugin refresh and agent-routing diff --git a/src/ucode/agents/claude.py b/src/ucode/agents/claude.py index c62380d27..2455f059e 100644 --- a/src/ucode/agents/claude.py +++ b/src/ucode/agents/claude.py @@ -1525,6 +1525,7 @@ def launch( *, options: LaunchOptions, ) -> None: + smart_routing_v2.cleanup_stale_claude_routing_plugins() tool_args = _with_debug_log(tool_args) binary = SPEC["binary"] workspace = state.get("workspace") diff --git a/src/ucode/smart_routing/claude_pty.py b/src/ucode/smart_routing/claude_pty.py index 88d07f0bb..f35c414ab 100644 --- a/src/ucode/smart_routing/claude_pty.py +++ b/src/ucode/smart_routing/claude_pty.py @@ -294,6 +294,7 @@ def run_claude_pty( model_switch_persisted: Callable[[], bool] = lambda: True, restore_model_setting: Callable[[], None] = lambda: None, log_path: Path | None = None, + pass_fds: tuple[int, ...] = (), ) -> int: """Run Claude in a PTY, switch its model, and replay the first prompt.""" @@ -334,6 +335,8 @@ def on_blocked_prompt(prompt: str, model: str) -> None: pid, master_fd = pty.fork() if pid == 0: + for descriptor in pass_fds: + os.set_inheritable(descriptor, True) os.execvp(argv[0], argv) os._exit(127) diff --git a/src/ucode/smart_routing/v2.py b/src/ucode/smart_routing/v2.py index 3858486a2..d26ee3ad0 100644 --- a/src/ucode/smart_routing/v2.py +++ b/src/ucode/smart_routing/v2.py @@ -3,6 +3,7 @@ import hashlib import json import os +import re import shutil import signal import socket @@ -11,7 +12,8 @@ import time import urllib.request import uuid -from collections.abc import Callable, MutableMapping +from collections.abc import Callable, Iterator, MutableMapping +from contextlib import contextmanager from pathlib import Path from typing import NoReturn, TextIO @@ -293,6 +295,65 @@ def _write_routed_claude_plugin(plugin_dir: Path, model_ids: list[str]) -> None: ) +def cleanup_stale_claude_routing_plugins() -> None: + """Remove abandoned launch plugins without touching active Claude sessions.""" + if os.name == "nt": + return + import fcntl + + for plugin_dir in APP_DIR.glob("claude-v2-*-plugin"): + match = re.fullmatch(r"claude-v2-([1-9][0-9]*)-[0-9a-f]{8}-plugin", plugin_dir.name) + if match is None or plugin_dir.is_symlink() or not plugin_dir.is_dir(): + continue + manifest = read_json_safe(plugin_dir / ".claude-plugin" / "plugin.json") + if manifest and manifest.get("name") != CLAUDE_ROUTING_PLUGIN_NAME: + continue + try: + os.kill(int(match[1]), 0) + except ProcessLookupError: + pass + except (OSError, OverflowError): + continue + else: + continue + lease_path = plugin_dir / ".lease" + try: + if lease_path.is_symlink(): + continue + if lease_path.exists(): + with lease_path.open("r+") as lease: + try: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + continue + shutil.rmtree(plugin_dir) + else: + shutil.rmtree(plugin_dir) + except FileNotFoundError: + pass + except OSError as exc: + print_warning(f"Could not remove stale Claude routing plugin {plugin_dir}: {exc}") + + +@contextmanager +def _routed_claude_plugin(plugin_dir: Path, model_ids: list[str]) -> Iterator[int]: + import fcntl + + plugin_dir.mkdir(parents=True) + try: + with (plugin_dir / ".lease").open("w") as lease: + fcntl.flock(lease, fcntl.LOCK_EX) + _write_routed_claude_plugin(plugin_dir, model_ids) + yield lease.fileno() + finally: + try: + shutil.rmtree(plugin_dir) + except FileNotFoundError: + pass + except OSError as exc: + print_warning(f"Could not remove Claude routing plugin {plugin_dir}: {exc}") + + def _request_claude_routing_decision( workspace: str, token: str, @@ -497,33 +558,6 @@ def launch_claude( sync_smart_routing_hooks(settings, routing_state, enabled=True) if route_first_prompt: sync_first_prompt_hook(settings, hook_executable) - write_json_file(settings_path, settings) - _write_routed_claude_plugin(plugin_dir, model_ids) - model_args = launch_model_args(remaining, launch_model) - argv = [ - binary, - "--settings", - str(settings_path), - *model_args, - "--plugin-dir", - str(plugin_dir), - *remaining, - ] - - if not route_first_prompt: - # Subagent-only routing needs no PTY: the PreToolUse hooks ride in the - # per-launch settings, so spawn Claude directly and clean up after it. - proc = subprocess.Popen(argv) - try: - returncode = proc.wait() - except KeyboardInterrupt: - proc.send_signal(signal.SIGINT) - returncode = proc.wait() - finally: - settings_path.unlink(missing_ok=True) - shutil.rmtree(plugin_dir, ignore_errors=True) - sys.exit(returncode) - model_setting = _ClaudeModelSettingGuard(user_settings_path) def route_prompt(prompt: str) -> claude_pty.FirstPromptRoute: @@ -535,20 +569,40 @@ def route_prompt(prompt: str) -> claude_pty.FirstPromptRoute: ) try: - returncode = claude_pty.run_claude_pty( - argv, - route_prompt=route_prompt, - socket_path=socket_path, - prepare_model_switch=model_setting.begin, - model_switch_persisted=model_setting.is_routed, - restore_model_setting=model_setting.restore, - log_path=CLAUDE_PTY_LOG, - ) + write_json_file(settings_path, settings) + with _routed_claude_plugin(plugin_dir, model_ids) as lease_fd: + model_args = launch_model_args(remaining, launch_model) + argv = [ + binary, + "--settings", + str(settings_path), + *model_args, + "--plugin-dir", + str(plugin_dir), + *remaining, + ] + if route_first_prompt: + returncode = claude_pty.run_claude_pty( + argv, + route_prompt=route_prompt, + socket_path=socket_path, + prepare_model_switch=model_setting.begin, + model_switch_persisted=model_setting.is_routed, + restore_model_setting=model_setting.restore, + log_path=CLAUDE_PTY_LOG, + pass_fds=(lease_fd,), + ) + else: + proc = subprocess.Popen(argv, pass_fds=(lease_fd,)) + try: + returncode = proc.wait() + except KeyboardInterrupt: + proc.send_signal(signal.SIGINT) + returncode = proc.wait() finally: model_setting.restore() settings_path.unlink(missing_ok=True) socket_path.unlink(missing_ok=True) - shutil.rmtree(plugin_dir, ignore_errors=True) sys.exit(returncode) diff --git a/tests/README.md b/tests/README.md index 7695cd135..cfbdd2265 100644 --- a/tests/README.md +++ b/tests/README.md @@ -29,6 +29,12 @@ precedence, stderr-only notices, actionable filesystem errors, and forwarding through normal, smart-routed, and relayed launches. These are component checks; they do not claim remote log retrieval or a live Claude session. +`TestRoutingPluginCleanup` in `test_claude_smart_routing_v2.py` checks disabled +launch cleanup, both enabled routing modes, setup/launch failures, and preservation +of active and unrelated plugins. Real subprocess/PTY checks verify that the plugin +lease survives exec and protects a Claude child after its parent closes the lease. +These component checks do not invoke model inference. + ## CUJ coverage matrix `test_repro_stale_claude_subagent.py` checks the diagnostic script's evidence diff --git a/tests/conftest.py b/tests/conftest.py index a1a7419ff..fd8dfd56f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,11 +34,13 @@ def _isolate_ucode_state(tmp_path, monkeypatch): import ucode.managed_files as managed_files_mod import ucode.state as state_mod from ucode.agents import codex as codex_mod + from ucode.smart_routing import v2 as smart_routing_v2 state_dir = tmp_path / ".ucode" state_dir.mkdir() monkeypatch.setattr(state_mod, "STATE_PATH", state_dir / "state.json") monkeypatch.setattr(config_io_mod, "APP_DIR", state_dir) + monkeypatch.setattr(smart_routing_v2, "APP_DIR", state_dir) # MANAGED_CONFIG_PATH is bound from APP_DIR at import, so patching APP_DIR alone doesn't move it; # rebind it or save_managed_state writes to the developer's real ~/.ucode/managed-config.json. monkeypatch.setattr( diff --git a/tests/test_claude_smart_routing_v2.py b/tests/test_claude_smart_routing_v2.py index 95c11c4e9..52b760431 100644 --- a/tests/test_claude_smart_routing_v2.py +++ b/tests/test_claude_smart_routing_v2.py @@ -4,6 +4,8 @@ import json import os +import select +import subprocess import sys import threading import time @@ -11,7 +13,7 @@ import pytest -from ucode.agents import claude +from ucode.agents import LaunchOptions, claude from ucode.databricks import AnthropicModelCatalog from ucode.smart_routing import claude_hooks, claude_pty, routing, v2 @@ -469,6 +471,142 @@ def send_signal(self, _signal): assert json.loads(user_settings.read_text()) == {"model": "opus"} +class TestRoutingPluginCleanup: + @staticmethod + def owner_exited(_pid, _signal): + raise ProcessLookupError + + @pytest.mark.parametrize("flag_value", [None, "0"]) + def test_disabled_launch_removes_abandoned_plugins(self, tmp_path, monkeypatch, flag_value): + monkeypatch.setattr(v2, "APP_DIR", tmp_path) + monkeypatch.setattr(v2.os, "kill", self.owner_exited) + for flag in (v2.ENABLE_SMART_ROUTING_ENV_VAR, v2.ENABLE_SUBAGENT_ROUTING_ENV_VAR): + if flag_value is None: + monkeypatch.delenv(flag, raising=False) + else: + monkeypatch.setenv(flag, flag_value) + stale = tmp_path / "claude-v2-123-deadbeef-plugin" + v2._write_routed_claude_plugin(stale, ["system.ai.glm-5-3"]) + captured = [] + monkeypatch.setattr(claude, "exec_or_spawn", captured.append) + + claude.launch({}, [], options=LaunchOptions()) + + assert not stale.exists() + assert len(captured) == 1 + assert "--plugin-dir" not in captured[0] + assert not list(tmp_path.glob("claude-v2-*-plugin")) + + def test_live_owner_and_unrelated_plugins_are_preserved(self, tmp_path, monkeypatch): + monkeypatch.setattr(v2, "APP_DIR", tmp_path) + live = tmp_path / f"claude-v2-{os.getpid()}-deadbeef-plugin" + v2._write_routed_claude_plugin(live, ["system.ai.glm-5-3"]) + unrelated = tmp_path / "claude-v2-123-12345678-plugin" + manifest = unrelated / ".claude-plugin/plugin.json" + manifest.parent.mkdir(parents=True) + manifest.write_text(json.dumps({"name": "user-plugin"})) + custom = tmp_path / "custom-plugin" + custom.mkdir() + link = tmp_path / "claude-v2-123-aaaaaaaa-plugin" + link.symlink_to(custom, target_is_directory=True) + + v2.cleanup_stale_claude_routing_plugins() + + assert live.exists() + assert unrelated.exists() + assert custom.exists() + assert link.is_symlink() + + def test_inherited_lease_protects_orphaned_claude(self, tmp_path, monkeypatch): + import fcntl + + monkeypatch.setattr(v2, "APP_DIR", tmp_path) + plugin = tmp_path / "claude-v2-123-deadbeef-plugin" + v2._write_routed_claude_plugin(plugin, ["system.ai.glm-5-3"]) + with (plugin / ".lease").open("w") as lease: + fcntl.flock(lease, fcntl.LOCK_EX) + child = subprocess.Popen( + [sys.executable, "-c", "import sys; print('ready', flush=True); sys.stdin.read()"], + pass_fds=(lease.fileno(),), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + try: + assert select.select([child.stdout], [], [], 5)[0], "child failed to start" + assert child.stdout.readline().strip() == "ready" + with monkeypatch.context() as scope: + scope.setattr(v2.os, "kill", self.owner_exited) + v2.cleanup_stale_claude_routing_plugins() + assert plugin.is_dir() + finally: + try: + child.communicate(timeout=5) + except subprocess.TimeoutExpired: + child.kill() + child.communicate(timeout=5) + with monkeypatch.context() as scope: + scope.setattr(v2.os, "kill", self.owner_exited) + v2.cleanup_stale_claude_routing_plugins() + assert not plugin.exists() + + @pytest.mark.parametrize("mode", ["full", "subagent"]) + @pytest.mark.parametrize("failure", [None, "plugin-write", "launch"]) + def test_plugin_lifetime_covers_setup_and_process_errors( + self, tmp_path, monkeypatch, mode, failure + ): + monkeypatch.setattr(v2, "APP_DIR", tmp_path) + monkeypatch.setenv(v2.ENABLE_SMART_ROUTING_ENV_VAR, "1" if mode == "full" else "0") + monkeypatch.setenv(v2.ENABLE_SUBAGENT_ROUTING_ENV_VAR, "1" if mode == "subagent" else "0") + monkeypatch.setattr(v2, "_launch_token", lambda *_args: "token") + monkeypatch.setattr(v2, "build_auth_token_argv", lambda *_args, **_kwargs: ["ug"]) + monkeypatch.setattr( + v2, + "_model_picker_catalog", + lambda: AnthropicModelCatalog( + model_ids=["system.ai.glm-5-3"], model_id_to_display_name={} + ), + ) + original_write = v2._write_routed_claude_plugin + + def write_plugin(path, models): + original_write(path, models) + if failure == "plugin-write": + raise RuntimeError("plugin write failed") + + def launch_process(argv, **kwargs): + plugin = Path(argv[argv.index("--plugin-dir") + 1]) + assert plugin.is_dir() + assert len(kwargs["pass_fds"]) == 1 + os.fstat(kwargs["pass_fds"][0]) + if failure == "launch": + raise RuntimeError("process launch failed") + return 0 + + class Process: + def __init__(self, argv, **kwargs): + launch_process(argv, **kwargs) + + def wait(self): + return 0 + + monkeypatch.setattr(v2, "_write_routed_claude_plugin", write_plugin) + monkeypatch.setattr(v2.subprocess, "Popen", Process) + monkeypatch.setattr(claude_pty, "run_claude_pty", launch_process) + with pytest.raises(RuntimeError if failure else SystemExit): + v2.launch_claude( + {"workspace": "https://example.com"}, + [], + binary="claude", + user_settings_path=tmp_path / "user-settings.json", + launch_model=None, + compose_settings=lambda args: ({}, args), + launch_model_args=claude._launch_model_args, + model_name=claude._maybe_add_1m_suffix, + ) + assert not list(tmp_path.glob("claude-v2-*")) + + class TestV2ModelPickerDiscovery: """modelPicker takes priority over gateway model discovery for smart routing.""" @@ -708,6 +846,25 @@ def run_second() -> None: class TestPtyFlow: + def test_passed_lease_descriptor_survives_exec(self, tmp_path): + capture = tmp_path / "descriptor.txt" + with (tmp_path / "lease").open("w") as lease: + result = claude_pty.run_claude_pty( + [ + sys.executable, + "-c", + "import os, sys; from pathlib import Path; " + "Path(sys.argv[2]).write_text(str(os.fstat(int(sys.argv[1])).st_ino))", + str(lease.fileno()), + str(capture), + ], + route_prompt=lambda _prompt: pytest.fail("no prompt expected"), + socket_path=tmp_path / "first.sock", + pass_fds=(lease.fileno(),), + ) + assert result == 0 + assert int(capture.read_text()) == os.fstat(lease.fileno()).st_ino + def test_does_not_launch_when_socket_startup_fails(self, tmp_path, monkeypatch): class StoppedThread: @staticmethod