From 846ac98f53019d0c74dfb64e7cb6190557c7d206 Mon Sep 17 00:00:00 2001 From: Garming Date: Sun, 20 Sep 2026 18:01:20 +0800 Subject: [PATCH 01/16] fix(studio): analyse migration routes on a background app-server driver Route analysis read its result out of the Codex text stream, so a progress update or a Markdown-fenced reply could be mistaken for the contract and fail the whole migration. Deliver it through a registered dynamic tool instead: the payload stays in typed JSON-RPC arguments, a rejected result returns as `success: false` so the same turn can correct itself, and a second prompt retries the contract when the first turn produces nothing usable. The turn also no longer runs inside the upload request. Studio starts a background worker, records a lease in `control/analysis-driver.json`, and keeps its heartbeat fresh while Codex works; a later read hands the attempt back to the in-Sandbox script once the lease goes stale, so a Studio restart cannot leave a task analysing forever. The scripted path stays as the fallback and `AGENTKIT_MIGRATION_APP_SERVER=0` pins it, which is what lets the app-server path run by default. - app_server: validate dynamic-tool arguments against the analysis contract, answer rejections with the failing field, and end the turn once the result is in - codex_app_server: register dynamic tools, retry overloaded turns, and keep a `willRetry` notice from ending the current turn - service: start both drivers behind one status contract, recover stalled app-server attempts on read, scan the scripted output for the newest contract JSON, and report diagnostics through the veadk logger (the Studio entrypoint pins the root logger to ERROR, which hid a silent fallback) --- frontend/server/migration/app_server.py | 147 ++++ frontend/server/migration/routes.py | 7 + frontend/server/migration/service.py | 703 +++++++++++++++++- tests/cli/test_codex_app_server.py | 291 ++++++++ .../test_migration_analysis_protocol.py | 320 ++++++++ tests/frontend/test_migration_app_server.py | 348 +++++++++ tests/frontend/test_migration_routes.py | 7 + tests/frontend/test_migration_server.py | 209 +++++- .../frontend/test_migration_service_edges.py | 10 + veadk/cli/codex_app_server.py | 172 ++++- 10 files changed, 2138 insertions(+), 76 deletions(-) create mode 100644 frontend/server/migration/app_server.py create mode 100644 tests/frontend/test_migration_analysis_protocol.py create mode 100644 tests/frontend/test_migration_app_server.py diff --git a/frontend/server/migration/app_server.py b/frontend/server/migration/app_server.py new file mode 100644 index 000000000..1c99be33c --- /dev/null +++ b/frontend/server/migration/app_server.py @@ -0,0 +1,147 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Codex app-server route analysis for Studio project migration. + +The migration analysis result is a strict contract object. Delivering it through a +registered dynamic tool keeps the payload in typed JSON-RPC arguments, so a progress +update, a commentary message, or a Markdown-fenced reply can no longer be mistaken for +the result. Rejections are returned to Codex as ``success: false`` so the same turn can +correct itself instead of failing the whole migration. +""" + +from __future__ import annotations + +from collections.abc import Callable +import os + +from veadk.cli.codex_app_server import ( + CodexAppServerError, + CodexAppServerSession, + CodexDynamicToolResult, +) + +from .contracts import MigrationContractError, validate_analysis_result + +ROUTE_TOOL_NAME = "reportRoute" +ROUTE_TOOL_DESCRIPTION = ( + "提交只读项目分析的最终结果。必须在完成分析后调用一次，" + "参数严格遵循给定的 JSON Schema；被拒绝时按返回的错误修正后重新调用。" +) +_APP_SERVER_ENV = "AGENTKIT_MIGRATION_APP_SERVER" +_DISABLED_VALUES = {"0", "false", "no", "off"} + + +def app_server_analysis_enabled() -> bool: + """Whether route analysis should use the Sandbox Codex app-server. + + The app-server carries the analysis result in typed dynamic-tool arguments, so a + progress update or a Markdown-fenced reply can no longer be mistaken for the + result. It runs on a Studio background worker, and the scripted ``codex exec`` + path remains the fallback whenever the app-server is unreachable, so this is on by + default; set ``AGENTKIT_MIGRATION_APP_SERVER=0`` to pin the scripted path. + """ + return os.getenv(_APP_SERVER_ENV, "").strip().lower() not in _DISABLED_VALUES + + +class MigrationAnalysisUnavailable(RuntimeError): + """The Sandbox app-server could not produce a usable analysis result.""" + + +class RouteRecorder: + """Validate and retain one analysis result delivered by a dynamic tool call.""" + + def __init__(self, *, attempt: int, input_sha256: str) -> None: + self.attempt = attempt + self.input_sha256 = input_sha256 + self.result: dict[str, object] | None = None + self.rejections: list[str] = [] + + def submit(self, arguments: dict[str, object]) -> CodexDynamicToolResult: + candidate = { + **arguments, + "attempt": self.attempt, + "input_sha256": self.input_sha256, + } + try: + validated = validate_analysis_result(candidate) + except MigrationContractError as error: + self.rejections.append(str(error)) + return CodexDynamicToolResult( + False, + f"分析结果不符合协议（{error}）。请修正后重新调用 {ROUTE_TOOL_NAME}。", + ) + if self.result is None: + self.result = validated + return CodexDynamicToolResult( + True, + "分析结果已接收。请用简体中文给出简短的用户可见总结。", + ) + + +async def run_route_analysis( + *, + endpoint: str, + prompt: str, + schema: dict[str, object], + cwd: str, + attempt: int, + input_sha256: str, + model: str = "", + timeout_seconds: float, + event_sink: Callable[[object], None] | None = None, +) -> dict[str, object] | None: + """Run one analysis turn and return the validated route contract, if any.""" + recorder = RouteRecorder(attempt=attempt, input_sha256=input_sha256) + session = CodexAppServerSession(endpoint) + session.cwd = cwd + if model: + session.model = model + session.register_dynamic_tool( + ROUTE_TOOL_NAME, + ROUTE_TOOL_DESCRIPTION, + schema, + recorder.submit, + ) + try: + await session.connect() + except CodexAppServerError as error: + raise MigrationAnalysisUnavailable(str(error)) from error + try: + async for event in session.stream_turn( + prompt, + timeout_seconds=timeout_seconds, + ): + if event_sink is not None: + event_sink(event) + if recorder.result is not None: + # 结果已经到手：终止本轮，避免继续消耗 token 和沙箱时间。 + await session.interrupt() + break + except CodexAppServerError as error: + if recorder.result is None: + raise MigrationAnalysisUnavailable(str(error)) from error + finally: + await session.close() + return recorder.result + + +__all__ = [ + "MigrationAnalysisUnavailable", + "ROUTE_TOOL_DESCRIPTION", + "ROUTE_TOOL_NAME", + "RouteRecorder", + "app_server_analysis_enabled", + "run_route_analysis", +] diff --git a/frontend/server/migration/routes.py b/frontend/server/migration/routes.py index 4f3bb3f1e..bc31b23eb 100644 --- a/frontend/server/migration/routes.py +++ b/frontend/server/migration/routes.py @@ -503,6 +503,13 @@ async def get_task( lambda: service.get_task(task_id, owner_id), task_id=task_id, ) + # A background app-server analysis dies with the Studio process; while a + # client keeps polling, hand its attempt back to the in-Sandbox script. + await invoke( + "recover_stalled_analysis", + lambda: service.recover_stalled_analysis(task_id, owner_id), + task_id=task_id, + ) decorated = await with_evaluation(task, owner_id) evaluation = decorated.get("evaluation") if isinstance(evaluation, dict) and evaluation.get("enabled") is True: diff --git a/frontend/server/migration/service.py b/frontend/server/migration/service.py index 6b8fa1d5a..7cd316355 100644 --- a/frontend/server/migration/service.py +++ b/frontend/server/migration/service.py @@ -16,14 +16,16 @@ from __future__ import annotations +import asyncio +import contextlib import hashlib import io import json -import logging import mimetypes import re import shlex import stat +import threading import time import uuid import zipfile @@ -37,6 +39,7 @@ from veadk.cli.studio_model_catalog import ( provider_allows_studio_development_model, ) +from veadk.utils.logger import get_logger from frontend.server.deployment_source import ( DeploymentSourceError, @@ -68,6 +71,11 @@ MigrationRemoteFileNotFound, MigrationSandboxSession, ) +from .app_server import ( + MigrationAnalysisUnavailable, + app_server_analysis_enabled, + run_route_analysis, +) from .models import ( MIGRATION_FRAMEWORKS, STRUCTURED_ENTRY_PATTERN, @@ -120,8 +128,43 @@ _ANALYSIS_STATUS_PATH = f"{MIGRATION_ROOT}/control/task-status.json" _ANALYSIS_RESULT_PATH = f"{MIGRATION_ROOT}/analysis/route.json" _ANALYSIS_PROMPT_PATH = f"{MIGRATION_ROOT}/analysis/prompt.md" +_ANALYSIS_RETRY_PROMPT_PATH = f"{MIGRATION_ROOT}/analysis/retry-prompt.md" _ANALYSIS_SCHEMA_PATH = f"{MIGRATION_ROOT}/analysis/route-schema.json" _ANALYSIS_PROCESS_EXIT_PATH = f"{MIGRATION_ROOT}/diagnostics/analysis/process-exit.json" +_ANALYSIS_EXTRACTION_DIAGNOSTICS_PATH = ( + f"{MIGRATION_ROOT}/diagnostics/analysis/result-extraction.json" +) +_ANALYSIS_CONTRACT_KEYS = ( + "schema_version", + "status", + "attempt", + "input_sha256", + "summary", + "frameworks", + "recommended", + "entries", + "boundary", + "assumptions", + "questions", + "warnings", +) +_ANALYSIS_CONTRACT_STATUSES = ("needs_input", "recommendation_ready", "unsupported") +_ANALYSIS_TURN_TIMEOUT_SECONDS = 600.0 +_ANALYSIS_DRIVER_PATH = f"{MIGRATION_ROOT}/control/analysis-driver.json" +_ANALYSIS_DRIVER_APP_SERVER = "app-server" +_ANALYSIS_DRIVER_SCRIPT = "codex-exec" +_ANALYSIS_DRIVER_RUNNING = "running" +_ANALYSIS_DRIVER_DONE = "done" +# 后台驱动的租约：心跳过期说明持有它的 Studio 进程已经不在了。 +_ANALYSIS_DRIVER_HEARTBEAT_SECONDS = 20.0 +_ANALYSIS_DRIVER_STALE_SECONDS = 90.0 +# 标识写入租约的后台驱动属于哪个 Studio 进程，便于诊断跨进程接管。 +_STUDIO_PROCESS_ID = uuid.uuid4().hex +_ANALYSIS_STATUS_MESSAGES = { + "ready": "项目分析完成，请确认迁移方式", + "needs_input": "需要补充少量信息后继续分析", +} +_ANALYSIS_UNSUPPORTED_MESSAGE = "当前项目不适用于已支持的迁移方式" _CONFIRMATION_PATH = f"{MIGRATION_ROOT}/control/route-selection.json" _INSTRUCTION_PATH = f"{MIGRATION_ROOT}/control/instruction.txt" _STOPPED_PATH = f"{MIGRATION_ROOT}/control/stopped.json" @@ -173,7 +216,10 @@ ) _ENV_REFERENCE_RE = re.compile(r"\$\{|\$\(|`") -logger = logging.getLogger(__name__) +# Anchor the analysis diagnostics under the veadk logger: the Studio entrypoint +# pins the root logger to ERROR, so a plain module logger would hide a silent +# fallback from the app-server path to the scripted one. +logger = get_logger(__name__) def _public_environment_defaults( @@ -1331,8 +1377,17 @@ def _analysis_prompt( input_sha256: str, previous_analysis: dict[str, object] | None = None, answers: dict[str, str] | None = None, + protocol_retry: bool = False, ) -> str: instruction = str(request.get("instruction") or "").strip() + retry_context = ( + "\n## 协议重试\n" + "上一次回复无法作为分析结果读取：其中没有符合输出协议的 JSON 对象。" + "请基于已经完成的分析重新给出结论，并且只输出那一个 JSON 对象，" + "不要输出 Markdown 围栏、进度说明、步骤清单或任何额外文字。\n" + if protocol_retry + else "" + ) previous_context = ( "\n".join( [ @@ -1480,6 +1535,7 @@ def _analysis_prompt( - 最终响应必须严格符合提供的 JSON Schema，只输出一个 JSON 对象，不要输出 Markdown 围栏、解释或额外文字。 +{retry_context} ## 用户补充要求 {instruction or "用户未补充额外要求。"} @@ -1630,39 +1686,174 @@ def is_macos_metadata(path): return "set -euo pipefail\npython3 - <<'PY'\n" + script.strip() + "\nPY" -def _codex_event_extractor() -> str: - return ( - "import json,sys\n" - "message = None\n" - "with open(sys.argv[1], encoding='utf-8') as events:\n" - " for line in events:\n" - " try:\n" - " event = json.loads(line)\n" - " except (TypeError, ValueError):\n" - " continue\n" - " item = event.get('item')\n" - " if (\n" - " event.get('type') == 'item.completed'\n" - " and isinstance(item, dict)\n" - " and item.get('type') == 'agent_message'\n" - " and isinstance(item.get('text'), str)\n" - " and item['text'].strip()\n" - " ):\n" - " message = item['text']\n" - "if message is None:\n" - " raise SystemExit('Codex agent_message event is missing')\n" - "with open(sys.argv[2], 'w', encoding='utf-8') as output:\n" - " output.write(message)\n" +def _analysis_result_extractor_script() -> str: + """Return the in-Sandbox script that recovers one analysis result object. + + Codex interleaves progress updates with its final answer, may deliver that answer as + commentary, and may wrap it in Markdown. Selecting the last agent message blindly + therefore fails whenever a progress update arrives last, which is exactly what the + analysis protocol asks Codex to emit. This script instead scans every agent message + from newest to oldest and keeps the first JSON object that satisfies the analysis + contract, so non-contract progress text is skipped instead of being fatal. + """ + return f""" +import json +import sys + +_CONTRACT_KEYS = {list(_ANALYSIS_CONTRACT_KEYS)!r} +_CONTRACT_STATUSES = {list(_ANALYSIS_CONTRACT_STATUSES)!r} +_NEWLINE = chr(10) + + +def _objects(text): + stripped = text.strip() + try: + value = json.loads(stripped) + except ValueError: + pass + else: + if isinstance(value, dict): + yield value + for block in stripped.split("```")[1::2]: + body = block.split(_NEWLINE, 1)[1] if _NEWLINE in block else "" + try: + value = json.loads(body.strip()) + except ValueError: + continue + if isinstance(value, dict): + yield value + decoder = json.JSONDecoder() + for index, character in enumerate(stripped): + if character != "{{": + continue + try: + value, _ = decoder.raw_decode(stripped[index:]) + except ValueError: + continue + if isinstance(value, dict): + yield value + + +def _contract(value): + if not isinstance(value, dict): + return None + if value.get("schema_version") != 1: + return None + if value.get("status") not in _CONTRACT_STATUSES: + return None + if any(key not in value for key in _CONTRACT_KEYS): + return None + return value + + +def main(argv): + if len(argv) < 3: + raise SystemExit("usage: extractor [diagnostics]") + answers = [] + commentary = [] + with open(argv[1], encoding="utf-8") as events: + for line in events: + try: + event = json.loads(line) + except (TypeError, ValueError): + continue + if not isinstance(event, dict) or event.get("type") != "item.completed": + continue + item = event.get("item") + if not isinstance(item, dict) or item.get("type") != "agent_message": + continue + text = item.get("text") + if not isinstance(text, str) or not text.strip(): + continue + if item.get("phase") == "commentary": + commentary.append(text) + else: + answers.append(text) + reason = ( + "no_agent_message" + if not answers and not commentary + else "no_contract_object" ) + found = None + for text in list(reversed(answers)) + list(reversed(commentary)): + for value in _objects(text): + contract = _contract(value) + if contract is not None: + found = contract + break + if found is not None: + break + if found is not None: + reason = "extracted" + if len(argv) > 3: + with open(argv[3], "w", encoding="utf-8") as diagnostics: + json.dump( + {{ + "reason": reason, + "answer_messages": len(answers), + "commentary_messages": len(commentary), + }}, + diagnostics, + ensure_ascii=False, + ) + if found is None: + raise SystemExit("Codex analysis result is unavailable: " + reason) + with open(argv[2], "w", encoding="utf-8") as output: + json.dump(found, output, ensure_ascii=False) -def _start_analysis_command(task_id: str, attempt: int) -> str: - running_status = { +if __name__ == "__main__": + main(sys.argv) +""" + + +def _analysis_running_status(attempt: int) -> dict[str, object]: + """The analysis status while Codex works, shared by both analysis drivers.""" + return { "schema_version": 1, "attempt": attempt, "state": "analyzing", "message": "正在分析项目框架、入口与迁移边界", } + + +def _clear_analysis_status_command() -> str: + """Return the command that drops driver state before a takeover start.""" + return "\n".join( + [ + "set -euo pipefail", + f"rm -f {shlex.quote(_ANALYSIS_STATUS_PATH)}", + f"rm -f {shlex.quote(_ANALYSIS_DRIVER_PATH)}", + ] + ) + + +def _analysis_driver_marker( + *, + driver: str, + attempt: int, + input_sha256: str = "", + started_at: float | None = None, + heartbeat_at: float | None = None, + owner_process: str = "", + state: str = _ANALYSIS_DRIVER_RUNNING, +) -> dict[str, object]: + """Describe which driver owns the analysis attempt currently in flight.""" + started = time.time() if started_at is None else started_at + return { + "schema_version": 1, + "driver": driver, + "state": state, + "attempt": attempt, + "input_sha256": input_sha256, + "started_at": started, + "heartbeat_at": started if heartbeat_at is None else heartbeat_at, + "owner_process": owner_process, + } + + +def _start_analysis_command(task_id: str, attempt: int) -> str: + running_status = _analysis_running_status(attempt) ready_status = { "schema_version": 1, "attempt": attempt, @@ -1686,6 +1877,17 @@ def _start_analysis_command(task_id: str, attempt: int) -> str: "retryable": False, }, } + protocol_failed_status = { + "schema_version": 1, + "attempt": attempt, + "state": "failed", + "message": "Codex 未返回可解析的分析结果，请重试", + "error": { + "code": "MIGRATION_ANALYSIS_RESULT_MISSING", + "message": "Codex 未产出符合分析协议的 JSON 结果。", + "retryable": True, + }, + } start_failed_status = { "schema_version": 1, "attempt": attempt, @@ -1723,22 +1925,49 @@ def _start_analysis_command(task_id: str, attempt: int) -> str: "import json,sys; " f"raise SystemExit(0 if json.load(open(sys.argv[1])).get('attempt') == {attempt} else 1)" ) - extract_agent_message = shlex.quote(_codex_event_extractor()) + extract_analysis_result = shlex.quote(_analysis_result_extractor_script()) + retry_log_path = ( + f"{MIGRATION_ROOT}/diagnostics/analysis/attempt-{attempt}-retry.log" + ) + extraction_diagnostics = f"{_ANALYSIS_EXTRACTION_DIAGNOSTICS_PATH}.{attempt}" inner = "\n".join( [ "set +e", + "run_analysis() {", ( - "codex exec --json --sandbox read-only --skip-git-repo-check " + " codex exec --json --sandbox read-only --skip-git-repo-check " f"--cd {shlex.quote(_PROJECT_PATH)} " f"--output-schema {shlex.quote(_ANALYSIS_SCHEMA_PATH)} " - f"- < {shlex.quote(_ANALYSIS_PROMPT_PATH)} " - f"> {shlex.quote(log_path)} 2>&1" + '- < "$1" > "$2" 2>&1' + ), + "}", + ( + f"run_analysis {shlex.quote(_ANALYSIS_PROMPT_PATH)} " + f"{shlex.quote(log_path)}" ), "code=$?", + "extracted=0", + ( + f"if python3 -c {extract_analysis_result} " + f"{shlex.quote(log_path)} {shlex.quote(result_tmp)} " + f"{shlex.quote(extraction_diagnostics)}; then extracted=1; fi" + ), + # 协议重试：上一轮回复无法作为分析结果读取时，在同一项目内再要一次纯 + # JSON 结论，避免一次格式偏差就让整个迁移任务失败。 + 'if [ "$extracted" -ne 1 ]; then', ( - f"if python3 -c {extract_agent_message} " - f"{shlex.quote(log_path)} {shlex.quote(result_tmp)} && " - f"python3 -c {validate_json} " + f" run_analysis {shlex.quote(_ANALYSIS_RETRY_PROMPT_PATH)} " + f"{shlex.quote(retry_log_path)}" + ), + " code=$?", + ( + f" if python3 -c {extract_analysis_result} " + f"{shlex.quote(retry_log_path)} {shlex.quote(result_tmp)} " + f"{shlex.quote(extraction_diagnostics)}; then extracted=1; fi" + ), + "fi", + ( + f'if [ "$extracted" -eq 1 ] && python3 -c {validate_json} ' f"{shlex.quote(result_tmp)}; then" ), ( @@ -1760,7 +1989,11 @@ def _start_analysis_command(task_id: str, attempt: int) -> str: "else", ' if [ "$code" -eq 0 ]; then code=1; fi', f" rm -f {shlex.quote(result_tmp)}", - f" {_atomic_json_command(_ANALYSIS_STATUS_PATH, failed_status)}", + ' if [ "$extracted" -eq 1 ]; then', + f" {_atomic_json_command(_ANALYSIS_STATUS_PATH, failed_status)}", + " else", + f" {_atomic_json_command(_ANALYSIS_STATUS_PATH, protocol_failed_status)}", + " fi", "fi", "finished_at=$(python3 -c 'import time; print(int(time.time()))')", ( @@ -2143,6 +2376,8 @@ def __init__( ) -> None: self._gateway = gateway self._clock = clock + # 进程内的后台分析驱动，键为 (session_id, attempt)，避免重复起同一轮分析。 + self._analysis_drivers: dict[tuple[str, int], threading.Thread] = {} @staticmethod def _translate(error: MigrationGatewayError) -> MigrationError: @@ -2715,13 +2950,377 @@ def upload_source( ).encode("utf-8"), media_type="text/markdown", ) + self._put( + session, + _ANALYSIS_RETRY_PROMPT_PATH, + _analysis_prompt( + request, + attempt=1, + input_sha256=digest, + protocol_retry=True, + ).encode("utf-8"), + media_type="text/markdown", + ) + if self._start_app_server_analysis( + session, + prompt=_analysis_prompt( + request, + attempt=1, + input_sha256=digest, + ), + attempt=1, + input_sha256=digest, + model_id=str(request.get("model_id") or ""), + ): + return self.get_task(task_id, owner_id) + self._start_scripted_analysis(session, task_id=task_id, attempt=1) + return self.get_task(task_id, owner_id) + + def _start_scripted_analysis( + self, + session: MigrationSandboxSession, + *, + task_id: str, + attempt: int, + clear_status: bool = False, + ) -> None: + """Run the analysis inside the Sandbox with ``codex exec``. + + The generated script owns its own background process, so Studio only launches + it. A takeover start (``clear_status``) first drops the state left by the + previous driver, because the script refuses to start when a status for the + same attempt already exists. + """ + if clear_status: + self._execute( + session, + _clear_analysis_status_command(), + operation="clear_analysis", + timeout_seconds=30, + ) + self._put( + session, + _ANALYSIS_DRIVER_PATH, + _json_bytes( + _analysis_driver_marker( + driver=_ANALYSIS_DRIVER_SCRIPT, + attempt=attempt, + owner_process=_STUDIO_PROCESS_ID, + ) + ), + media_type="application/json", + ) self._execute( session, - _start_analysis_command(task_id, 1), + _start_analysis_command(task_id, attempt), operation="start_analysis", timeout_seconds=30, ) - return self.get_task(task_id, owner_id) + + def _start_app_server_analysis( + self, + session: MigrationSandboxSession, + *, + prompt: str, + attempt: int, + input_sha256: str, + model_id: str = "", + timeout_seconds: float = _ANALYSIS_TURN_TIMEOUT_SECONDS, + ) -> bool: + """Analyse through the Sandbox app-server on a Studio background worker. + + The turn must not run inside the HTTP request: an upload that waits for Codex + would be cut off by the gateway on a long analysis. The worker keeps the same + file contract as the scripted path, so ``get_task`` reads both drivers alike. + Returns ``False`` when the caller must start the scripted path instead. + """ + if not app_server_analysis_enabled(): + return False + key = (session.session_id, attempt) + running = self._analysis_drivers.get(key) + if running is not None and running.is_alive(): + return True + started_at = time.time() + self._put( + session, + _ANALYSIS_DRIVER_PATH, + _json_bytes( + _analysis_driver_marker( + driver=_ANALYSIS_DRIVER_APP_SERVER, + attempt=attempt, + input_sha256=input_sha256, + started_at=started_at, + owner_process=_STUDIO_PROCESS_ID, + ) + ), + media_type="application/json", + ) + self._put( + session, + _ANALYSIS_STATUS_PATH, + _json_bytes(_analysis_running_status(attempt)), + media_type="application/json", + ) + worker = threading.Thread( + target=self._app_server_analysis_worker, + args=(session, attempt, input_sha256, prompt, model_id, timeout_seconds), + name=f"migration-analysis-{attempt}", + daemon=True, + ) + self._analysis_drivers[key] = worker + try: + worker.start() + except Exception: # noqa: BLE001 - a failed start must fall back to the script + self._analysis_drivers.pop(key, None) + logger.exception( + "Studio migration analysis worker could not start task_id=%s", + session.task_id, + ) + return False + return True + + def _app_server_analysis_worker( + self, + session: MigrationSandboxSession, + attempt: int, + input_sha256: str, + prompt: str, + model_id: str, + timeout_seconds: float, + ) -> None: + """Run one app-server turn and persist it, or hand over to the script.""" + key = (session.session_id, attempt) + try: + try: + analysis = asyncio.run( + self._run_app_server_turn( + session, + attempt=attempt, + input_sha256=input_sha256, + prompt=prompt, + model_id=model_id, + timeout_seconds=timeout_seconds, + ) + ) + except MigrationAnalysisUnavailable as error: + logger.warning( + "Studio migration app-server analysis unavailable task_id=%s " + "attempt=%s error_type=%s", + session.task_id, + attempt, + type(error).__name__, + ) + analysis = None + if analysis is None: + self._start_scripted_analysis( + session, + task_id=session.task_id, + attempt=attempt, + clear_status=True, + ) + return + self._persist_app_server_analysis( + session, + attempt=attempt, + analysis=analysis, + ) + except Exception: # noqa: BLE001 - the worker must never kill the process + logger.exception( + "Studio migration app-server analysis worker failed task_id=%s " + "attempt=%s", + session.task_id, + attempt, + ) + finally: + if self._analysis_drivers.get(key) is threading.current_thread(): + self._analysis_drivers.pop(key, None) + + async def _run_app_server_turn( + self, + session: MigrationSandboxSession, + *, + attempt: int, + input_sha256: str, + prompt: str, + model_id: str, + timeout_seconds: float, + ) -> dict[str, object] | None: + """Run the app-server turn while refreshing the background driver lease.""" + + async def beat() -> None: + warned = False + while True: + await asyncio.sleep(_ANALYSIS_DRIVER_HEARTBEAT_SECONDS) + try: + await asyncio.to_thread( + self._put, + session, + _ANALYSIS_DRIVER_PATH, + _json_bytes( + _analysis_driver_marker( + driver=_ANALYSIS_DRIVER_APP_SERVER, + attempt=attempt, + input_sha256=input_sha256, + owner_process=_STUDIO_PROCESS_ID, + ) + ), + media_type="application/json", + ) + except Exception as error: # noqa: BLE001 - lease refresh is advisory + if not warned: + warned = True + logger.warning( + "Studio migration analysis lease refresh failed " + "task_id=%s error_type=%s", + session.task_id, + type(error).__name__, + ) + + heartbeat = asyncio.create_task(beat()) + try: + return await run_route_analysis( + endpoint=session.endpoint, + prompt=prompt, + schema=_analysis_schema(), + cwd=_PROJECT_PATH, + attempt=attempt, + input_sha256=input_sha256, + model=model_id, + timeout_seconds=timeout_seconds, + ) + finally: + heartbeat.cancel() + with contextlib.suppress(asyncio.CancelledError): + await heartbeat + + def _persist_app_server_analysis( + self, + session: MigrationSandboxSession, + *, + attempt: int, + analysis: dict[str, object], + ) -> None: + """Store the contract delivered by the dynamic tool and close the lease.""" + status = str(analysis.get("status") or "") + if status == "unsupported": + payload: dict[str, object] = { + "schema_version": 1, + "attempt": attempt, + "state": "failed", + "message": _ANALYSIS_UNSUPPORTED_MESSAGE, + "error": { + "code": "MIGRATION_ANALYSIS_UNSUPPORTED", + "message": "项目分析未找到可执行的迁移方式。", + "retryable": False, + }, + } + else: + payload = { + "schema_version": 1, + "attempt": attempt, + "state": "ready" if status == "recommendation_ready" else status, + "message": _ANALYSIS_STATUS_MESSAGES.get( + "ready" if status == "recommendation_ready" else status, + "项目分析已更新", + ), + } + self._put( + session, + _ANALYSIS_RESULT_PATH, + _json_bytes(analysis), + media_type="application/json", + ) + self._put( + session, + _ANALYSIS_STATUS_PATH, + _json_bytes(payload), + media_type="application/json", + ) + self._put( + session, + _ANALYSIS_DRIVER_PATH, + _json_bytes( + _analysis_driver_marker( + driver=_ANALYSIS_DRIVER_APP_SERVER, + attempt=attempt, + owner_process=_STUDIO_PROCESS_ID, + state=_ANALYSIS_DRIVER_DONE, + ) + ), + media_type="application/json", + ) + logger.info( + "Studio migration app-server analysis completed task_id=%s " + "attempt=%s status=%s", + session.task_id, + attempt, + status, + ) + + def recover_stalled_analysis(self, task_id: str, owner_id: str) -> bool: + """Hand a stalled app-server analysis back to the scripted driver. + + A Studio restart drops the worker that owned the turn while the task still + reads as analysing. The lease written by the worker says who owns it and how + fresh it is, so a request may take over once that lease goes stale. + """ + try: + session = self._session(task_id, owner_id) + marker = self._read_json( + session, + _ANALYSIS_DRIVER_PATH, + optional=True, + ) + except Exception as error: # noqa: BLE001 - recovery must never fail a read + logger.warning( + "Studio migration analysis recovery skipped task_id=%s error_type=%s", + task_id, + type(error).__name__, + ) + return False + if not isinstance(marker, dict): + return False + if str(marker.get("driver") or "") != _ANALYSIS_DRIVER_APP_SERVER: + return False + if str(marker.get("state") or _ANALYSIS_DRIVER_RUNNING) != ( + _ANALYSIS_DRIVER_RUNNING + ): + return False + attempt = marker.get("attempt") + if not isinstance(attempt, int) or attempt < 1: + return False + running = self._analysis_drivers.get((session.session_id, attempt)) + if running is not None and running.is_alive(): + return False + heartbeat = marker.get("heartbeat_at") + age = ( + time.time() - float(heartbeat) + if isinstance(heartbeat, (int, float)) + else _ANALYSIS_DRIVER_STALE_SECONDS + ) + if age < _ANALYSIS_DRIVER_STALE_SECONDS: + return False + logger.warning( + "Studio migration app-server analysis lease expired; restarting the " + "scripted driver task_id=%s attempt=%s", + task_id, + attempt, + ) + try: + self._start_scripted_analysis( + session, + task_id=task_id, + attempt=attempt, + clear_status=True, + ) + except Exception as error: # noqa: BLE001 - recovery must never fail a read + logger.warning( + "Studio migration analysis recovery failed task_id=%s error_type=%s", + task_id, + type(error).__name__, + ) + return False + return True def list_tasks(self, owner_id: str) -> dict[str, list[dict[str, object]]]: try: @@ -3323,12 +3922,34 @@ def submit_answers( ).encode("utf-8"), media_type="text/markdown", ) - self._execute( + self._put( session, - _start_analysis_command(task_id, next_attempt), - operation="start_analysis", - timeout_seconds=30, + _ANALYSIS_RETRY_PROMPT_PATH, + _analysis_prompt( + request, + attempt=next_attempt, + input_sha256=str(source["sha256"]), + previous_analysis=analysis, + answers=body.answers, + protocol_retry=True, + ).encode("utf-8"), + media_type="text/markdown", ) + if self._start_app_server_analysis( + session, + prompt=_analysis_prompt( + request, + attempt=next_attempt, + input_sha256=str(source["sha256"]), + previous_analysis=analysis, + answers=body.answers, + ), + attempt=next_attempt, + input_sha256=str(source["sha256"]), + model_id=str(request.get("model_id") or ""), + ): + return self.get_task(task_id, owner_id) + self._start_scripted_analysis(session, task_id=task_id, attempt=next_attempt) return self.get_task(task_id, owner_id) def confirm( diff --git a/tests/cli/test_codex_app_server.py b/tests/cli/test_codex_app_server.py index 430beea5f..d92c04a7e 100644 --- a/tests/cli/test_codex_app_server.py +++ b/tests/cli/test_codex_app_server.py @@ -28,6 +28,7 @@ from veadk.cli import codex_app_server from veadk.cli.codex_app_server import ( CodexAppServerError, + CodexAppServerOverloadError, CodexAppServerSession, CodexAppServerTransportError, CodexAppServerTurnTimeoutError, @@ -1307,6 +1308,32 @@ async def test_workspace_directory_browsing_and_user_approval() -> None: await session.close() +@pytest.mark.parametrize( + "method", ["mcpServer/elicitation/request", "item/tool/requestUserInput"] +) +@pytest.mark.asyncio +async def test_unimplemented_input_protocols_fail_closed(method: str) -> None: + """Characterize the current integration gap before implementing the feature.""" + websocket = _FakeWebSocket() + session = CodexAppServerSession( + "https://sandbox.example", + websocket_factory=lambda _url: _ready(websocket), + ) + await session.connect() + try: + await session._handle_server_request( + "input-probe", method, {"threadId": "thread-1"} + ) + response = next(m for m in websocket.messages if m.get("id") == "input-probe") + assert response["error"] == { + "code": -32601, + "message": f"unsupported server request: {method}", + } + assert "result" not in response + finally: + await session.close() + + @pytest.mark.asyncio async def test_closed_transport_reconnects_and_resumes_active_thread() -> None: first = _FakeWebSocket() @@ -2782,3 +2809,267 @@ async def test_dynamic_tools_register_only_on_start_and_survive_resume(): assert session.dynamic_tools == (tool,) finally: await session.close() + + +class _DynamicToolWebSocket(_FakeWebSocket): + """Issue one ``item/tool/call`` server request while a turn is running.""" + + def __init__(self, *, tool: str = "reportRoute") -> None: + super().__init__() + self.tool = tool + self.tool_result: dict[str, object] | None = None + self.tool_error: dict[str, object] | None = None + + async def send(self, raw: str) -> None: + message = json.loads(raw) + if message.get("id") == "server-tool-call": + if "result" in message: + self.tool_result = message["result"] + else: + self.tool_error = message["error"] + await self._notification( + "turn/completed", + {"turn": {"id": "turn-1", "status": "completed"}}, + ) + return + await super().send(raw) + if message.get("method") == "turn/start": + await self.queue.put( + json.dumps( + { + "id": "server-tool-call", + "method": "item/tool/call", + "params": { + "threadId": "thread-1", + "turnId": "turn-1", + "callId": "call-1", + "tool": self.tool, + "arguments": {"status": "recommendation_ready"}, + }, + } + ) + ) + + +class _OverloadedWebSocket(_FakeWebSocket): + """Reject the first ``failures`` requests with the queue-overload code.""" + + def __init__(self, *, failures: int) -> None: + super().__init__() + self.remaining_failures = failures + self.overload_count = 0 + + async def send(self, raw: str) -> None: + message = json.loads(raw) + request_id = message.get("id") + if ( + isinstance(request_id, int) + and message.get("method") not in {None, "initialize"} + and self.remaining_failures > 0 + ): + self.remaining_failures -= 1 + self.overload_count += 1 + self.messages.append(message) + await self.queue.put( + json.dumps( + { + "id": request_id, + "error": { + "code": -32001, + "message": "Server overloaded; retry later.", + }, + } + ) + ) + return + await super().send(raw) + + +def test_dynamic_tools_are_announced_only_when_a_thread_starts() -> None: + session = CodexAppServerSession("https://sandbox.example?Authorization=secret") + + session.register_dynamic_tool( + "reportRoute", + "提交项目分析结果", + {"type": "object"}, + lambda _arguments: codex_app_server.CodexDynamicToolResult(True, "已接收"), + ) + + assert session.dynamic_tool_names == ("reportRoute",) + assert session._thread_start_options()["dynamicTools"] == [ + { + "type": "function", + "name": "reportRoute", + "description": "提交项目分析结果", + "inputSchema": {"type": "object"}, + } + ] + # thread/resume 不接受 dynamicTools，只有 thread/start 携带。 + assert "dynamicTools" not in session._thread_options() + + +def test_dynamic_tool_registration_rejects_duplicates_and_missing_fields() -> None: + session = CodexAppServerSession("https://sandbox.example?Authorization=secret") + + def handler( + _arguments: dict[str, object], + ) -> codex_app_server.CodexDynamicToolResult: + return codex_app_server.CodexDynamicToolResult(True, "ok") + + with pytest.raises(CodexAppServerError, match="必须提供名称与说明"): + session.register_dynamic_tool("", "说明", {}, handler) + with pytest.raises(CodexAppServerError, match="JSON Schema"): + session.register_dynamic_tool("a", "说明", "not-a-schema", handler) # type: ignore[arg-type] + + session.register_dynamic_tool("reportRoute", "说明", {"type": "object"}, handler) + with pytest.raises(CodexAppServerError, match="已注册"): + session.register_dynamic_tool( + "reportRoute", "说明", {"type": "object"}, handler + ) + + +@pytest.mark.asyncio +async def test_dynamic_tool_call_returns_a_typed_result() -> None: + websocket = _DynamicToolWebSocket() + seen: list[dict[str, object]] = [] + session = CodexAppServerSession( + "https://sandbox.example?Authorization=secret", + websocket_factory=lambda _url: _ready(websocket), + ) + session.register_dynamic_tool( + "reportRoute", + "提交项目分析结果", + {"type": "object"}, + lambda arguments: ( + seen.append(arguments) + or codex_app_server.CodexDynamicToolResult(True, "分析结果已接收") + ), + ) + await session.connect() + + events = [event async for event in session.stream_turn("analyze")] + + assert seen == [{"status": "recommendation_ready"}] + assert websocket.tool_result == { + "contentItems": [{"type": "inputText", "text": "分析结果已接收"}], + "success": True, + } + assert [event.text for event in events if event.text] == ["完成"] + await session.close() + + +@pytest.mark.asyncio +async def test_dynamic_tool_failure_is_reported_to_codex_without_aborting_the_turn() -> ( + None +): + websocket = _DynamicToolWebSocket() + session = CodexAppServerSession( + "https://sandbox.example?Authorization=secret", + websocket_factory=lambda _url: _ready(websocket), + ) + session.register_dynamic_tool( + "reportRoute", + "提交项目分析结果", + {"type": "object"}, + lambda _arguments: codex_app_server.CodexDynamicToolResult( + False, "frameworks[0].evidence[2].line 必须为大于 0 的整数" + ), + ) + await session.connect() + + async for _event in session.stream_turn("analyze"): + pass + + assert websocket.tool_result is not None + assert websocket.tool_result["success"] is False + assert "evidence" in websocket.tool_result["contentItems"][0]["text"] + assert websocket.tool_result["contentItems"][0]["text"].startswith("frameworks") + await session.close() + + +@pytest.mark.asyncio +async def test_an_unregistered_dynamic_tool_is_refused() -> None: + websocket = _DynamicToolWebSocket(tool="unknownTool") + session = CodexAppServerSession( + "https://sandbox.example?Authorization=secret", + websocket_factory=lambda _url: _ready(websocket), + ) + await session.connect() + + async for _event in session.stream_turn("analyze"): + pass + + assert websocket.tool_error is None + assert websocket.tool_result == { + "success": False, + "contentItems": [ + {"type": "inputText", "text": "Tool is unavailable for this thread."} + ], + } + await session.close() + + +@pytest.mark.asyncio +async def test_a_handler_exception_becomes_a_failed_tool_result() -> None: + websocket = _DynamicToolWebSocket() + session = CodexAppServerSession( + "https://sandbox.example?Authorization=secret", + websocket_factory=lambda _url: _ready(websocket), + ) + + def _explode(_arguments: dict[str, object]) -> object: + raise RuntimeError("handler blew up") + + session.register_dynamic_tool( + "reportRoute", "提交项目分析结果", {"type": "object"}, _explode + ) + await session.connect() + + async for _event in session.stream_turn("analyze"): + pass + + assert websocket.tool_result is not None + assert websocket.tool_result["success"] is False + text = websocket.tool_result["contentItems"][0]["text"] + assert "RuntimeError" not in text + assert "handler blew up" not in text + await session.close() + + +@pytest.mark.asyncio +async def test_queue_overload_is_retried_with_backoff( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(codex_app_server, "_OVERLOAD_RETRY_BASE_SECONDS", 0.0) + websocket = _OverloadedWebSocket(failures=2) + session = CodexAppServerSession( + "https://sandbox.example?Authorization=secret", + websocket_factory=lambda _url: _ready(websocket), + ) + await session.connect() + + result = await session.request("thread/read", {"threadId": "thread-1"}) + + assert websocket.overload_count == 2 + assert "thread" in result + await session.close() + + +@pytest.mark.asyncio +async def test_queue_overload_gives_up_after_the_retry_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(codex_app_server, "_OVERLOAD_RETRY_BASE_SECONDS", 0.0) + websocket = _OverloadedWebSocket(failures=0) + session = CodexAppServerSession( + "https://sandbox.example?Authorization=secret", + websocket_factory=lambda _url: _ready(websocket), + ) + await session.connect() + websocket.remaining_failures = 99 + + with pytest.raises(CodexAppServerOverloadError): + await session.request("thread/read", {"threadId": "thread-1"}) + + assert websocket.overload_count == codex_app_server._OVERLOAD_RETRY_ATTEMPTS + 1 + await session.close() diff --git a/tests/frontend/test_migration_analysis_protocol.py b/tests/frontend/test_migration_analysis_protocol.py new file mode 100644 index 000000000..b65a755ab --- /dev/null +++ b/tests/frontend/test_migration_analysis_protocol.py @@ -0,0 +1,320 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end checks for the in-Sandbox analysis protocol. + +These tests execute the real shell command Studio ships to the Dev Sandbox against a +stub ``codex`` binary, so the extraction, protocol retry, and status transitions are +verified as a whole rather than as isolated units. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import time + +import pytest + +from frontend.server.migration import service as migration_service + + +def _contract(status: str = "recommendation_ready") -> dict[str, object]: + return { + "schema_version": 1, + "status": status, + "attempt": 1, + "input_sha256": "a" * 64, + "summary": "项目分析摘要", + "frameworks": [], + "recommended": ( + None + if status == "unsupported" + else {"framework": "dify", "entry": None, "reason": "理由"} + ), + "entries": [], + "boundary": {"include": [], "exclude": []}, + "assumptions": [], + "questions": [], + "warnings": [], + } + + +def _message(text: str, phase: str | None = None) -> dict[str, object]: + item: dict[str, object] = {"type": "agent_message", "text": text} + if phase is not None: + item["phase"] = phase + return {"type": "item.completed", "item": item} + + +class AnalysisSandbox: + """One temporary Sandbox-like root driving the real analysis command.""" + + def __init__(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + self.root = tmp_path / "migration" + self.bin = tmp_path / "bin" + self.bin.mkdir(parents=True, exist_ok=True) + self.plan_path = tmp_path / "plan.json" + monkeypatch.setenv("FAKE_CODEX_PLAN", str(self.plan_path)) + for name, value in ( + ("MIGRATION_ROOT", str(self.root)), + ("_PROJECT_PATH", f"{self.root}/workspace/source"), + ("_ANALYSIS_RESULT_PATH", f"{self.root}/analysis/route.json"), + ("_ANALYSIS_PROMPT_PATH", f"{self.root}/analysis/prompt.md"), + ("_ANALYSIS_RETRY_PROMPT_PATH", f"{self.root}/analysis/retry-prompt.md"), + ("_ANALYSIS_SCHEMA_PATH", f"{self.root}/analysis/route-schema.json"), + ("_ANALYSIS_STATUS_PATH", f"{self.root}/control/task-status.json"), + ( + "_ANALYSIS_PROCESS_EXIT_PATH", + f"{self.root}/diagnostics/analysis/process-exit.json", + ), + ( + "_ANALYSIS_EXTRACTION_DIAGNOSTICS_PATH", + f"{self.root}/diagnostics/analysis/result-extraction.json", + ), + ): + monkeypatch.setattr(migration_service, name, value) + for relative in ( + "workspace/source", + "analysis", + "control", + "diagnostics/analysis", + ): + (self.root / relative).mkdir(parents=True, exist_ok=True) + Path(migration_service._ANALYSIS_PROMPT_PATH).write_text( + "分析提示词", encoding="utf-8" + ) + Path(migration_service._ANALYSIS_RETRY_PROMPT_PATH).write_text( + "## 协议重试\n只输出 JSON", encoding="utf-8" + ) + Path(migration_service._ANALYSIS_SCHEMA_PATH).write_text("{}", encoding="utf-8") + self._install_stub() + + def _install_stub(self) -> None: + stub = self.bin / "codex" + stub.write_text( + "#!/usr/bin/env python3\n" + "import json, os, sys\n" + "plan = json.load(open(os.environ['FAKE_CODEX_PLAN'], encoding='utf-8'))\n" + "prompt = sys.stdin.read()\n" + "events = plan['retry'] if '协议重试' in prompt else plan['first']\n" + "if events is None:\n" + " raise SystemExit(3)\n" + "for event in events:\n" + " print(json.dumps(event, ensure_ascii=False))\n", + encoding="utf-8", + ) + stub.chmod(0o755) + + def plan(self, *, first: object, retry: object = None) -> None: + self.plan_path.write_text( + json.dumps({"first": first, "retry": retry}, ensure_ascii=False), + encoding="utf-8", + ) + + def run(self, attempt: int = 1) -> None: + command = migration_service._start_analysis_command( + "migration-v1-" + "b" * 32, + attempt, + ) + environment = { + **os.environ, + "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", + } + completed = subprocess.run( + ["bash", "-c", command], + capture_output=True, + text=True, + env=environment, + check=False, + ) + assert completed.returncode == 0, completed.stderr + + def wait_for_status(self, timeout: float = 20.0) -> dict[str, object]: + status_path = self.root / "control/task-status.json" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if status_path.is_file(): + value = json.loads(status_path.read_text(encoding="utf-8")) + if value.get("state") != "analyzing": + return value + time.sleep(0.05) + raise AssertionError("analysis status was never written") + + def route(self) -> dict[str, object] | None: + path = self.root / "analysis/route.json" + if not path.is_file(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + def extraction_diagnostics(self) -> dict[str, object] | None: + path = Path(migration_service._ANALYSIS_EXTRACTION_DIAGNOSTICS_PATH + ".1") + if not path.is_file(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.mark.skipif( + shutil.which("setsid") is None or shutil.which("bash") is None, + reason="the analysis protocol needs bash and setsid", +) +def test_analysis_survives_a_trailing_progress_update( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression: the reported failure had a progress update as the last message.""" + sandbox = AnalysisSandbox(tmp_path, monkeypatch) + sandbox.plan( + first=[ + _message("正在扫描项目结构", phase="commentary"), + _message(json.dumps(_contract(), ensure_ascii=False)), + _message("已完成步骤 2：确认迁移边界", phase="commentary"), + ] + ) + + sandbox.run() + + status = sandbox.wait_for_status() + assert status["state"] == "ready" + assert sandbox.route() is not None + assert sandbox.route()["status"] == "recommendation_ready" # type: ignore[index] + assert sandbox.extraction_diagnostics() == { + "reason": "extracted", + "answer_messages": 1, + "commentary_messages": 2, + } + + +@pytest.mark.skipif( + shutil.which("setsid") is None or shutil.which("bash") is None, + reason="the analysis protocol needs bash and setsid", +) +def test_analysis_recovers_a_fenced_result_and_a_needs_input_status( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sandbox = AnalysisSandbox(tmp_path, monkeypatch) + contract = _contract("needs_input") + contract["questions"] = [{"id": "q1", "prompt": "请选择入口", "required": True}] + sandbox.plan( + first=[ + _message( + "结论：\n```json\n" + + json.dumps(contract, ensure_ascii=False) + + "\n```\n以上。" + ) + ] + ) + + sandbox.run() + + status = sandbox.wait_for_status() + assert status["state"] == "needs_input" + assert sandbox.route() is not None + + +@pytest.mark.skipif( + shutil.which("setsid") is None or shutil.which("bash") is None, + reason="the analysis protocol needs bash and setsid", +) +def test_analysis_retries_in_turn_when_the_first_reply_has_no_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sandbox = AnalysisSandbox(tmp_path, monkeypatch) + sandbox.plan( + first=[_message("我完成了分析，但没有输出 JSON。")], + retry=[_message(json.dumps(_contract(), ensure_ascii=False))], + ) + + sandbox.run() + + status = sandbox.wait_for_status() + assert status["state"] == "ready" + assert sandbox.route() is not None + + +@pytest.mark.skipif( + shutil.which("setsid") is None or shutil.which("bash") is None, + reason="the analysis protocol needs bash and setsid", +) +def test_analysis_reports_a_retryable_protocol_failure_after_both_attempts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sandbox = AnalysisSandbox(tmp_path, monkeypatch) + sandbox.plan( + first=[_message("没有 JSON。")], + retry=[_message("仍然没有 JSON。")], + ) + + sandbox.run() + + status = sandbox.wait_for_status() + assert status["state"] == "failed" + assert status["error"]["code"] == "MIGRATION_ANALYSIS_RESULT_MISSING" + assert status["error"]["retryable"] is True + assert sandbox.route() is None + assert sandbox.extraction_diagnostics() == { + "reason": "no_contract_object", + "answer_messages": 1, + "commentary_messages": 0, + } + + +@pytest.mark.skipif( + shutil.which("setsid") is None or shutil.which("bash") is None, + reason="the analysis protocol needs bash and setsid", +) +def test_analysis_rejects_an_event_stream_without_any_agent_message( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sandbox = AnalysisSandbox(tmp_path, monkeypatch) + sandbox.plan( + first=[{"type": "item.completed", "item": {"type": "reasoning", "text": "x"}}], + retry=[{"type": "item.completed", "item": {"type": "reasoning", "text": "y"}}], + ) + + sandbox.run() + + status = sandbox.wait_for_status() + assert status["state"] == "failed" + assert status["error"]["code"] == "MIGRATION_ANALYSIS_RESULT_MISSING" + assert sandbox.extraction_diagnostics()["reason"] == "no_agent_message" # type: ignore[index] + + +@pytest.mark.skipif( + shutil.which("setsid") is None or shutil.which("bash") is None, + reason="the analysis protocol needs bash and setsid", +) +def test_analysis_ignores_a_result_object_without_the_contract( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An unrelated JSON object must not be mistaken for the analysis result.""" + sandbox = AnalysisSandbox(tmp_path, monkeypatch) + sandbox.plan( + first=[_message(json.dumps({"attempt": 1}, ensure_ascii=False))], + retry=[_message(json.dumps(_contract("unsupported"), ensure_ascii=False))], + ) + + sandbox.run() + + status = sandbox.wait_for_status() + assert status["state"] == "failed" + assert status["error"]["code"] == "MIGRATION_ANALYSIS_UNSUPPORTED" diff --git a/tests/frontend/test_migration_app_server.py b/tests/frontend/test_migration_app_server.py new file mode 100644 index 000000000..50e23c4e7 --- /dev/null +++ b/tests/frontend/test_migration_app_server.py @@ -0,0 +1,348 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the optional Codex app-server route-analysis path.""" + +from __future__ import annotations + +import asyncio +import json +import threading +import time + +import pytest + +from frontend.server.migration import service as migration_service +from frontend.server.migration.app_server import ( + MigrationAnalysisUnavailable, + RouteRecorder, + app_server_analysis_enabled, +) +from frontend.server.migration.gateway import MigrationSandboxSession +from frontend.server.migration.service import MigrationService + + +def _session() -> MigrationSandboxSession: + return MigrationSandboxSession( + tool_id="tool-dev", + session_id="session-1", + task_id="migration-v1-" + "c" * 32, + endpoint="https://sandbox.invalid", + region="cn-beijing", + status="Ready", + created_at="2099-01-01T00:00:00Z", + expire_at="2099-01-01T01:00:00Z", + owner_id="owner", + ) + + +def _contract(status: str = "recommendation_ready") -> dict[str, object]: + return { + "schema_version": 1, + "status": status, + "attempt": 1, + "input_sha256": "a" * 64, + "summary": "摘要", + "frameworks": [], + "recommended": ( + None + if status == "unsupported" + else {"framework": "dify", "entry": None, "reason": "理由"} + ), + "entries": [], + "boundary": {"include": [], "exclude": []}, + "assumptions": [], + "questions": [], + "warnings": [], + } + + +class _Recorder: + """Minimal service-side recorder for the app-server path.""" + + def __init__(self) -> None: + self.written: dict[str, object] = {} + self.commands: list[str] = [] + + def put_file(self, session, path, content, *, media_type): + self.written[path] = json.loads(content) + + +class _Service(MigrationService): + def __init__(self, recorder: _Recorder, *, marker: dict[str, object] | None = None): + self._recorder = recorder + self._analysis_drivers = {} + self._marker = marker + self._session_marker = marker + + def _put(self, session, path, content, *, media_type): + self._recorder.put_file(session, path, content, media_type=media_type) + + def _execute(self, session, command, *, operation, timeout_seconds=120): + self._recorder.commands.append(command) + return {} + + def _session(self, task_id, owner_id): + return _session() + + def _read_json(self, session, path, *, optional=False): + return self._session_marker + + +def _wait_for_driver(service: MigrationService, *, timeout: float = 10.0) -> None: + deadline = time.monotonic() + timeout + while service._analysis_drivers and time.monotonic() < deadline: + time.sleep(0.01) + assert not service._analysis_drivers, "分析后台驱动没有结束" + + +def test_app_server_analysis_is_enabled_by_default_and_can_be_pinned_off( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AGENTKIT_MIGRATION_APP_SERVER", raising=False) + assert app_server_analysis_enabled() is True + + monkeypatch.setenv("AGENTKIT_MIGRATION_APP_SERVER", "0") + assert app_server_analysis_enabled() is False + + recorder = _Recorder() + service = _Service(recorder) + assert ( + service._start_app_server_analysis( + _session(), + prompt="分析", + attempt=1, + input_sha256="a" * 64, + ) + is False + ) + assert recorder.written == {} + assert service._analysis_drivers == {} + + +def test_app_server_analysis_runs_on_a_background_worker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AGENTKIT_MIGRATION_APP_SERVER", raising=False) + recorded: dict[str, object] = {} + release = threading.Event() + + async def _run(**kwargs: object) -> dict[str, object]: + recorded.update(kwargs) + await asyncio.to_thread(release.wait, 5) + return _contract() + + monkeypatch.setattr(migration_service, "run_route_analysis", _run) + + recorder = _Recorder() + service = _Service(recorder) + session = _session() + + assert ( + service._start_app_server_analysis( + session, + prompt="分析", + attempt=1, + input_sha256="a" * 64, + model_id="doubao-test", + ) + is True + ) + + # 上传请求立刻返回：此刻只有「正在分析」和租约，结果还没写。 + assert recorder.written[migration_service._ANALYSIS_STATUS_PATH] == { + "schema_version": 1, + "attempt": 1, + "state": "analyzing", + "message": "正在分析项目框架、入口与迁移边界", + } + lease = recorder.written[migration_service._ANALYSIS_DRIVER_PATH] + assert lease["driver"] == "app-server" + assert lease["state"] == "running" + assert lease["attempt"] == 1 + assert migration_service._ANALYSIS_RESULT_PATH not in recorder.written + + release.set() + _wait_for_driver(service) + + assert recorded["attempt"] == 1 + assert recorded["model"] == "doubao-test" + assert recorded["cwd"] == migration_service._PROJECT_PATH + assert recorded["schema"] == migration_service._analysis_schema() + assert recorder.written[migration_service._ANALYSIS_RESULT_PATH]["status"] == ( + "recommendation_ready" + ) + assert recorder.written[migration_service._ANALYSIS_STATUS_PATH] == { + "schema_version": 1, + "attempt": 1, + "state": "ready", + "message": "项目分析完成，请确认迁移方式", + } + assert recorder.written[migration_service._ANALYSIS_DRIVER_PATH]["state"] == "done" + + +def test_app_server_analysis_persists_an_unsupported_result( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AGENTKIT_MIGRATION_APP_SERVER", raising=False) + + async def _run(**_kwargs: object) -> dict[str, object]: + return _contract("unsupported") + + monkeypatch.setattr(migration_service, "run_route_analysis", _run) + + recorder = _Recorder() + service = _Service(recorder) + + assert service._start_app_server_analysis( + _session(), prompt="分析", attempt=1, input_sha256="a" * 64 + ) + _wait_for_driver(service) + + assert recorder.written[migration_service._ANALYSIS_STATUS_PATH] == { + "schema_version": 1, + "attempt": 1, + "state": "failed", + "message": "当前项目不适用于已支持的迁移方式", + "error": { + "code": "MIGRATION_ANALYSIS_UNSUPPORTED", + "message": "项目分析未找到可执行的迁移方式。", + "retryable": False, + }, + } + + +def test_app_server_analysis_falls_back_to_the_scripted_driver( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AGENTKIT_MIGRATION_APP_SERVER", raising=False) + + async def _run(**_kwargs: object) -> dict[str, object]: + raise MigrationAnalysisUnavailable( + "无法连接 AgentKit Session 中的 Codex 服务。" + ) + + monkeypatch.setattr(migration_service, "run_route_analysis", _run) + + recorder = _Recorder() + service = _Service(recorder) + session = _session() + + assert service._start_app_server_analysis( + session, prompt="分析", attempt=1, input_sha256="a" * 64 + ) + _wait_for_driver(service) + + assert recorder.commands[0] == migration_service._clear_analysis_status_command() + assert "ANALYSIS_STARTED_V1" in recorder.commands[1] + assert recorder.written[migration_service._ANALYSIS_DRIVER_PATH]["driver"] == ( + "codex-exec" + ) + + +@pytest.mark.parametrize( + "marker", + [ + {"driver": "codex-exec", "state": "running", "attempt": 1, "heartbeat_at": 0.0}, + {"driver": "app-server", "state": "done", "attempt": 1, "heartbeat_at": 0.0}, + ], +) +def test_analysis_recovery_ignores_turns_that_are_not_a_stalled_app_server( + marker: dict[str, object], +) -> None: + recorder = _Recorder() + service = _Service(recorder, marker=marker) + + assert ( + service.recover_stalled_analysis("migration-v1-" + "c" * 32, "owner") is False + ) + assert recorder.commands == [] + + +def test_analysis_recovery_waits_for_a_fresh_lease() -> None: + recorder = _Recorder() + service = _Service( + recorder, + marker={ + "driver": "app-server", + "state": "running", + "attempt": 1, + "heartbeat_at": time.time(), + }, + ) + + assert ( + service.recover_stalled_analysis("migration-v1-" + "c" * 32, "owner") is False + ) + assert recorder.commands == [] + + +def test_a_stalled_app_server_analysis_is_handed_back_to_the_script() -> None: + recorder = _Recorder() + service = _Service( + recorder, + marker={ + "driver": "app-server", + "state": "running", + "attempt": 1, + "heartbeat_at": time.time() - 10_000, + }, + ) + + assert service.recover_stalled_analysis("migration-v1-" + "c" * 32, "owner") is True + assert recorder.commands[0] == migration_service._clear_analysis_status_command() + assert "ANALYSIS_STARTED_V1" in recorder.commands[1] + + +def test_a_live_in_process_worker_blocks_analysis_recovery() -> None: + class _Alive: + @staticmethod + def is_alive() -> bool: + return True + + recorder = _Recorder() + service = _Service( + recorder, + marker={ + "driver": "app-server", + "state": "running", + "attempt": 1, + "heartbeat_at": time.time() - 10_000, + }, + ) + session = _session() + service._analysis_drivers[(session.session_id, 1)] = _Alive() + + assert ( + service.recover_stalled_analysis("migration-v1-" + "c" * 32, "owner") is False + ) + assert recorder.commands == [] + + +def test_route_recorder_rejects_an_invalid_contract_then_accepts_a_correction() -> None: + recorder = RouteRecorder(attempt=1, input_sha256="a" * 64) + + rejected = recorder.submit({"schema_version": 1, "status": "recommendation_ready"}) + + assert rejected.success is False + assert "reportRoute" in rejected.text + assert recorder.result is None + assert recorder.rejections + + accepted = recorder.submit(_contract()) + + assert accepted.success is True + assert recorder.result is not None + assert recorder.result["attempt"] == 1 + assert recorder.result["input_sha256"] == "a" * 64 diff --git a/tests/frontend/test_migration_routes.py b/tests/frontend/test_migration_routes.py index 903d4c812..37181484e 100644 --- a/tests/frontend/test_migration_routes.py +++ b/tests/frontend/test_migration_routes.py @@ -68,6 +68,10 @@ def upload_source( def get_task(self, task_id: str, owner_id: str) -> dict[str, object]: return self.record("get_task", task_id, owner_id) + def recover_stalled_analysis(self, task_id: str, owner_id: str) -> bool: + self.calls.append(("recover_stalled_analysis", (task_id, owner_id))) + return False + def submit_answers( self, task_id: str, @@ -272,6 +276,8 @@ def test_all_migration_routes_delegate_with_owner_and_return_artifacts() -> None "create_task", "upload_source", "get_task", + # 每次读取任务都会顺带检查是否要接管停滞的后台分析。 + "recover_stalled_analysis", "submit_answers", "confirm", "stop", @@ -471,6 +477,7 @@ def attach( assert stopped.json()["evaluation"]["error"]["code"] == expected_code assert [name for name, _ in service.calls] == [ "get_task", + "recover_stalled_analysis", "get_task", "stop", "artifact", diff --git a/tests/frontend/test_migration_server.py b/tests/frontend/test_migration_server.py index 680168246..03201f85e 100644 --- a/tests/frontend/test_migration_server.py +++ b/tests/frontend/test_migration_server.py @@ -14,11 +14,14 @@ from __future__ import annotations +import asyncio import hashlib import io import json import stat import subprocess +import threading +import time import zipfile from dataclasses import replace from datetime import datetime, timezone @@ -40,6 +43,7 @@ CreateMigrationTaskBody, SubmitAnalysisAnswersBody, ) +from frontend.server.migration import service as migration_service from frontend.server.migration.routes import mount_migration_routes from frontend.server.migration.service import ( EVALUATION_SESSION_TTL_SECONDS, @@ -53,7 +57,7 @@ _activity_secret_values, _analysis_result_message, _start_analysis_command, - _codex_event_extractor, + _analysis_result_extractor_script, _parse_activity_log, _public_environment_defaults, validate_source_archive, @@ -61,6 +65,16 @@ from veadk.cli.frontend_skill_creator import _sandbox_model_config +@pytest.fixture(autouse=True) +def _pin_scripted_analysis(monkeypatch: pytest.MonkeyPatch) -> None: + """These suites cover the in-Sandbox ``codex exec`` driver. + + The app-server driver has its own suite and its own background-worker test; pin + the scripted driver here so every command assertion stays deterministic. + """ + monkeypatch.setenv("AGENTKIT_MIGRATION_APP_SERVER", "0") + + def source_zip( files: dict[str, bytes] | None = None, *, @@ -2758,6 +2772,127 @@ def test_source_archive_validation_accepts_projects_and_rejects_unsafe_entries() validate_source_archive(b"not-a-zip") +def test_upload_starts_the_app_server_analysis_on_a_background_worker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """默认走 App Server：上传立刻返回，分析在后台线程完成后落盘。""" + monkeypatch.delenv("AGENTKIT_MIGRATION_APP_SERVER", raising=False) + release = threading.Event() + source = source_zip() + digest = hashlib.sha256(source).hexdigest() + + async def _run(**_kwargs: object) -> dict[str, object]: + await asyncio.to_thread(release.wait, 10) + return { + "schema_version": 1, + "status": "recommendation_ready", + "attempt": 1, + "input_sha256": digest, + "summary": "这是一个 LangChain 客服智能体。", + "frameworks": [ + { + "id": "langchain", + "confidence": "high", + "evidence": [ + {"path": "agent.py", "line": 1, "reason": "导入 Runnables。"} + ], + } + ], + "recommended": { + "framework": "langchain", + "entry": None, + "reason": "入口为模块级 Agent 对象。", + }, + "entries": [], + "boundary": {"include": ["agent.py"], "exclude": []}, + "assumptions": [], + "questions": [], + "warnings": [], + } + + monkeypatch.setattr(migration_service, "run_route_analysis", _run) + + gateway = FakeMigrationGateway() + service = MigrationService(gateway) + + created = service.create_task( + CreateMigrationTaskBody( + sourceFileName="support-agent.zip", + instruction="请保留客服流程，并使用中文输出迁移报告。", + ), + "owner-1", + "Owner", + ) + task_id = str(created["id"]) + uploaded = service.upload_source(task_id, "owner-1", source) + + # 请求没有被 Codex 阻塞：仍然在读「正在分析」，由后台线程去完成这一轮。 + assert uploaded["state"] == "analyzing" + assert [operation for _, operation, _ in gateway.commands] == [ + "accept_request", + "preflight", + "prepare_source", + ] + status = json.loads( + gateway.files[(task_id, f"{MIGRATION_ROOT}/control/task-status.json")] + ) + assert status == { + "schema_version": 1, + "attempt": 1, + "state": "analyzing", + "message": "正在分析项目框架、入口与迁移边界", + } + lease = json.loads( + gateway.files[(task_id, f"{MIGRATION_ROOT}/control/analysis-driver.json")] + ) + assert lease["driver"] == "app-server" + assert lease["state"] == "running" + assert f"{MIGRATION_ROOT}/analysis/route.json" not in dict(gateway.files) + + release.set() + deadline = time.monotonic() + 10 + while service._analysis_drivers and time.monotonic() < deadline: + time.sleep(0.01) + assert not service._analysis_drivers + + task = service.get_task(task_id, "owner-1") + assert task["state"] == "analysis_ready" + assert task["analysis"]["status"] == "recommendation_ready" + assert task["analysis"]["frameworks"][0]["id"] == "langchain" + + +def test_a_stalled_app_server_analysis_is_recovered_on_the_next_read( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Studio 重启后租约过期：下一次读取任务时交回沙箱内的脚本继续分析。""" + monkeypatch.delenv("AGENTKIT_MIGRATION_APP_SERVER", raising=False) + gateway = FakeMigrationGateway() + service = MigrationService(gateway) + task_id, _uploaded = create_uploaded_task(service) + service._analysis_drivers.clear() + + gateway.files[(task_id, f"{MIGRATION_ROOT}/control/analysis-driver.json")] = ( + json.dumps( + { + "schema_version": 1, + "driver": "app-server", + "state": "running", + "attempt": 1, + "input_sha256": hashlib.sha256(source_zip()).hexdigest(), + "started_at": 1.0, + "heartbeat_at": 1.0, + "owner_process": "gone", + } + ).encode("utf-8") + ) + + assert service.recover_stalled_analysis(task_id, "owner-1") is True + operations = [operation for _, operation, _ in gateway.commands] + assert operations[-2:] == ["clear_analysis", "start_analysis"] + assert "codex exec" in gateway.commands[-1][2] + assert service.recover_stalled_analysis(task_id, "owner-1") is False + + def test_upload_starts_read_only_codex_analysis_without_cli_inspection() -> None: gateway = FakeMigrationGateway() service = MigrationService(gateway) @@ -2890,11 +3025,26 @@ def test_upload_starts_read_only_codex_analysis_without_cli_inspection() -> None assert schema["properties"]["warnings"]["maxItems"] == 100 -def test_codex_analysis_uses_the_last_completed_agent_message( +def test_codex_analysis_selects_the_result_that_matches_the_contract( tmp_path: Path, ) -> None: events = tmp_path / "events.jsonl" result = tmp_path / "result.json" + diagnostics = tmp_path / "diagnostics.json" + contract = { + "schema_version": 1, + "status": "recommendation_ready", + "attempt": 1, + "input_sha256": "a" * 64, + "summary": "摘要", + "frameworks": [], + "recommended": {"framework": "dify", "entry": None, "reason": "理由"}, + "entries": [], + "boundary": {"include": [], "exclude": []}, + "assumptions": [], + "questions": [], + "warnings": [], + } events.write_text( "\n".join( [ @@ -2905,16 +3055,34 @@ def test_codex_analysis_uses_the_last_completed_agent_message( "item": {"type": "reasoning", "text": "ignored"}, } ), + # 进度更新不是结果，必须被跳过而不是当成结果。 + json.dumps( + { + "type": "item.completed", + "item": { + "type": "agent_message", + "phase": "commentary", + "text": "已完成步骤 1：扫描项目结构", + }, + } + ), json.dumps( { "type": "item.completed", - "item": {"type": "agent_message", "text": '{"attempt": 1}'}, + "item": { + "type": "agent_message", + "text": json.dumps(contract, ensure_ascii=False), + }, } ), json.dumps( { "type": "item.completed", - "item": {"type": "agent_message", "text": '{"attempt": 2}'}, + "item": { + "type": "agent_message", + "phase": "commentary", + "text": "已完成步骤 2：确认迁移边界", + }, } ), ] @@ -2923,14 +3091,28 @@ def test_codex_analysis_uses_the_last_completed_agent_message( ) extracted = subprocess.run( - ["python3", "-c", _codex_event_extractor(), str(events), str(result)], + [ + "python3", + "-c", + _analysis_result_extractor_script(), + str(events), + str(result), + str(diagnostics), + ], capture_output=True, check=False, text=True, ) assert extracted.returncode == 0, extracted.stderr - assert json.loads(result.read_text(encoding="utf-8")) == {"attempt": 2} + assert json.loads(result.read_text(encoding="utf-8"))["status"] == ( + "recommendation_ready" + ) + assert json.loads(diagnostics.read_text(encoding="utf-8")) == { + "reason": "extracted", + "answer_messages": 1, + "commentary_messages": 2, + } def test_codex_analysis_accepts_a_valid_final_message_after_nonzero_cli_exit() -> None: @@ -2947,6 +3129,7 @@ def test_codex_analysis_rejects_an_event_stream_without_an_agent_message( ) -> None: events = tmp_path / "events.jsonl" result = tmp_path / "result.json" + diagnostics = tmp_path / "diagnostics.json" events.write_text( json.dumps( { @@ -2958,15 +3141,25 @@ def test_codex_analysis_rejects_an_event_stream_without_an_agent_message( ) extracted = subprocess.run( - ["python3", "-c", _codex_event_extractor(), str(events), str(result)], + [ + "python3", + "-c", + _analysis_result_extractor_script(), + str(events), + str(result), + str(diagnostics), + ], capture_output=True, check=False, text=True, ) assert extracted.returncode != 0 - assert "agent_message event is missing" in extracted.stderr + assert "no_agent_message" in extracted.stderr assert not result.exists() + assert json.loads(diagnostics.read_text(encoding="utf-8"))["reason"] == ( + "no_agent_message" + ) def test_upload_can_resume_analysis_start_after_source_was_accepted() -> None: diff --git a/tests/frontend/test_migration_service_edges.py b/tests/frontend/test_migration_service_edges.py index 99ab91f4b..b3944b8a5 100644 --- a/tests/frontend/test_migration_service_edges.py +++ b/tests/frontend/test_migration_service_edges.py @@ -50,6 +50,16 @@ ) +@pytest.fixture(autouse=True) +def _pin_scripted_analysis(monkeypatch: pytest.MonkeyPatch) -> None: + """These suites cover the in-Sandbox ``codex exec`` driver. + + The app-server driver has its own suite and its own background-worker test; pin + the scripted driver here so every command assertion stays deterministic. + """ + monkeypatch.setenv("AGENTKIT_MIGRATION_APP_SERVER", "0") + + def zip_bytes(files: dict[str, bytes]) -> bytes: output = io.BytesIO() with zipfile.ZipFile(output, "w") as archive: diff --git a/veadk/cli/codex_app_server.py b/veadk/cli/codex_app_server.py index 16c8e382d..efb5341c6 100644 --- a/veadk/cli/codex_app_server.py +++ b/veadk/cli/codex_app_server.py @@ -24,6 +24,8 @@ import asyncio import base64 import contextlib +import inspect +import random import hashlib import json import logging @@ -71,6 +73,9 @@ ) _GATEWAY_WEBSOCKET_MAX_LIFETIME_SECONDS = 30 * 60 _GATEWAY_WEBSOCKET_REFRESH_MARGIN_SECONDS = 30 +_OVERLOAD_ERROR_CODE = -32001 +_OVERLOAD_RETRY_ATTEMPTS = 3 +_OVERLOAD_RETRY_BASE_SECONDS = 0.5 _TURN_FINAL_READ_ATTEMPTS = 4 _TURN_FINAL_READ_RETRY_SECONDS = 0.1 _ACTIVE_TURN_TRANSPORT_RECOVERY_ATTEMPTS = 2 @@ -101,6 +106,10 @@ class CodexAppServerTurnTimeoutError(CodexAppServerError): """A Codex turn exceeded its configured inactivity timeout.""" +class CodexAppServerOverloadError(CodexAppServerRequestError): + """The app-server inbound queue is full; the request is safe to retry.""" + + def _app_server_error_detail(error: object) -> str: """Preserve the complete JSON-RPC error payload for upstream diagnostics.""" if isinstance(error, (dict, list)): @@ -427,6 +436,20 @@ class CodexAppServerEvent: duration_ms: int | None = None +@dataclass(frozen=True) +class CodexDynamicToolResult: + """One dynamic tool outcome returned to Codex for a ``item/tool/call`` request.""" + + success: bool + text: str + + +DynamicToolHandler = Callable[ + [dict[str, object]], + "CodexDynamicToolResult | Awaitable[CodexDynamicToolResult]", +] + + class CodexAppServerSession: """Persistent JSON-RPC connection for one AgentKit cloud Session.""" @@ -459,6 +482,8 @@ def __init__( self._turn_final_item_id = "" self._reasoning_delta_text: dict[str, dict[int, str]] = {} self._item_phases: dict[str, str] = {} + self._dynamic_tool_specs: list[dict[str, object]] = [] + self._dynamic_tool_handlers: dict[str, DynamicToolHandler] = {} self._skills_by_id: dict[str, _CodexPrivateSkill] = {} self._skills_cwd = "" self._skills_loaded = False @@ -559,7 +584,9 @@ async def connect(self) -> None: else: snapshot = await self._request( "thread/start", - self._thread_start_options() if self.dynamic_tools else {}, + self._thread_start_options() + if self._announced_dynamic_tools() + else {}, ) self._apply_thread_snapshot(snapshot) except Exception: @@ -724,21 +751,30 @@ async def _request( raise CodexAppServerError("Codex app-server 尚未连接。") request_id = self._next_request_id self._next_request_id += 1 - future = asyncio.get_running_loop().create_future() - self._pending_requests[request_id] = future - try: - await self._send( - { - "id": request_id, - "method": method, - **({"params": params} if params is not None else {}), - } - ) - return await asyncio.wait_for(future, timeout=timeout) - except TimeoutError as error: - raise CodexAppServerError(f"Codex 操作 {method} 响应超时。") from error - finally: - self._pending_requests.pop(request_id, None) + message = { + "id": request_id, + "method": method, + **({"params": params} if params is not None else {}), + } + overloads = 0 + while True: + future = asyncio.get_running_loop().create_future() + self._pending_requests[request_id] = future + try: + await self._send(message) + return await asyncio.wait_for(future, timeout=timeout) + except CodexAppServerOverloadError: + # 服务端入站队列满时返回 -32001，属于可重试过载：按指数退避加抖动 + # 重试同一个请求，而不是把过载当成业务失败抛给用户。 + if overloads >= _OVERLOAD_RETRY_ATTEMPTS: + raise + delay = _OVERLOAD_RETRY_BASE_SECONDS * (2**overloads) + await asyncio.sleep(delay + random.uniform(0, delay)) + overloads += 1 + except TimeoutError as error: + raise CodexAppServerError(f"Codex 操作 {method} 响应超时。") from error + finally: + self._pending_requests.pop(request_id, None) async def notify( self, method: str, params: dict[str, object] | None = None @@ -1939,6 +1975,9 @@ def _handle_response(self, message: dict[str, object]) -> None: return error = message.get("error") if error is not None: + if isinstance(error, dict) and error.get("code") == _OVERLOAD_ERROR_CODE: + future.set_exception(CodexAppServerOverloadError(error)) + return future.set_exception(CodexAppServerRequestError(error)) return result = message.get("result") @@ -2120,11 +2159,20 @@ def _handle_notification(self, method: str, params: dict[str, object]) -> None: ) return message = params.get("message") - if ( - isinstance(message, str) - and self._turn_completion is not None - and not self._turn_completion.done() - ): + if not isinstance(message, str): + return + if params.get("willRetry") is True: + # 服务端仍会重试这一轮：只向前端报告可恢复的告警，不终止本轮。 + self._emit( + CodexAppServerEvent( + kind="warning", + item_id=_string(params.get("turnId"), 200), + status="running", + text=message, + ) + ) + return + if self._turn_completion is not None and not self._turn_completion.done(): self._turn_completion.set_exception(CodexAppServerError(message)) if method == "thread/status/changed": status = params.get("status") @@ -2280,10 +2328,16 @@ async def _handle_server_request( ) ) + def _announced_dynamic_tools(self) -> tuple[dict[str, object], ...]: + """Every tool spec announced to Codex on ``thread/start``.""" + return (*self.dynamic_tools, *self._dynamic_tool_specs) + async def _handle_dynamic_tool( self, request_id: object, params: dict[str, object] ) -> None: websocket = self._websocket + tool = params.get("tool") + handler = self._dynamic_tool_handlers.get(tool if isinstance(tool, str) else "") valid = ( params.get("threadId") == self.thread_id and all( @@ -2292,17 +2346,26 @@ async def _handle_dynamic_tool( ) and params.get("namespace") is None and any( - tool.get("name") == params.get("tool") for tool in self.dynamic_tools + spec.get("name") == tool for spec in self._announced_dynamic_tools() ) - and self.dynamic_tool_handler is not None + and (handler is not None or self.dynamic_tool_handler is not None) ) result: dict[str, object] if not valid: result = self._tool_failure("Tool is unavailable for this thread.") else: try: - assert self.dynamic_tool_handler is not None - result = await asyncio.wait_for(self.dynamic_tool_handler(params), 15) + if handler is not None: + arguments = params.get("arguments") + outcome = handler(arguments if isinstance(arguments, dict) else {}) + if inspect.isawaitable(outcome): + outcome = await outcome + result = self._tool_result(outcome) + else: + assert self.dynamic_tool_handler is not None + result = await asyncio.wait_for( + self.dynamic_tool_handler(params), 15 + ) except Exception as error: # Never expose callback exceptions (which may contain credentials). logger.warning( @@ -2325,6 +2388,20 @@ async def _handle_dynamic_tool( except (CodexAppServerTransportError, TimeoutError): logger.info("Codex dynamic tool reply pending reconnect") + @staticmethod + def _tool_result(outcome: object) -> dict[str, object]: + """Normalise one registered tool outcome into a JSON-RPC tool result.""" + if isinstance(outcome, CodexDynamicToolResult): + return { + "success": outcome.success, + "contentItems": [{"type": "inputText", "text": outcome.text}], + } + if isinstance(outcome, dict) and "contentItems" in outcome: + return outcome + return CodexAppServerSession._tool_failure( + "Tool returned an unsupported result." + ) + @staticmethod def _tool_failure(message: str) -> dict[str, object]: return { @@ -2477,6 +2554,46 @@ def _ensure_thread_idle(self, action: str) -> None: if not self.thread_id: raise CodexAppServerError("Codex Thread 尚未初始化。") + def register_dynamic_tool( + self, + name: str, + description: str, + input_schema: dict[str, object], + handler: DynamicToolHandler, + ) -> None: + """Register one client-side tool that Codex may call during a turn. + + The tool is announced through the experimental ``dynamicTools`` field of + ``thread/start``, so registration only reaches a thread that has not been + started yet. Handlers run in Studio and return a typed result, which keeps + structured outcomes out of the model's free-form text. + """ + if not name or not description: + raise CodexAppServerError("动态工具必须提供名称与说明。") + if not isinstance(input_schema, dict): + raise CodexAppServerError("动态工具必须提供 JSON Schema 参数定义。") + if self.thread_id and not self.workspace_locked: + raise CodexAppServerError("动态工具必须在 Codex Thread 开始前注册。") + if name in self._dynamic_tool_handlers: + raise CodexAppServerError(f"动态工具 {name} 已注册。") + self._dynamic_tool_handlers[name] = handler + self._dynamic_tool_specs.append( + { + "type": "function", + "name": name, + "description": description, + "inputSchema": input_schema, + } + ) + + @property + def dynamic_tool_names(self) -> tuple[str, ...]: + return tuple( + str(spec["name"]) + for spec in self._dynamic_tool_specs + if isinstance(spec.get("name"), str) + ) + def _thread_options(self) -> dict[str, object]: return { **({"cwd": self.cwd} if self.cwd else {}), @@ -2485,7 +2602,8 @@ def _thread_options(self) -> dict[str, object]: } def _thread_start_options(self) -> dict[str, object]: - if self.dynamic_tools: + announced = self._announced_dynamic_tools() + if announced: # thread/start uses sandbox (a mode), turn/start uses sandboxPolicy. return { **({"cwd": self.cwd} if self.cwd else {}), @@ -2493,7 +2611,7 @@ def _thread_start_options(self) -> dict[str, object]: "approvalPolicy": self.permissions.approval_policy, "approvalsReviewer": self.permissions.approvals_reviewer, "sandbox": self.permissions.sandbox_mode, - "dynamicTools": list(self.dynamic_tools), + "dynamicTools": list(announced), } return self._thread_options() From 13d3553f6948fbcfbfe311fa0ee2f71df2808fad Mon Sep 17 00:00:00 2001 From: Garming Date: Sun, 20 Sep 2026 23:25:20 +0800 Subject: [PATCH 02/16] fix(studio): judge migration batches on a dynamic-tool app-server turn Evaluation judging no longer parses the last `codex exec` agent message: the runner writes a durable `judge/request.json` for the batch it is judging and Studio answers it with one Codex app-server turn that returns the verdict through the `reportEvaluation` dynamic tool. The runner keeps owning batching, the thread record, the batch cache and the report, and still validates the verdict before caching it, so a rejected batch falls back to `codex exec` inside the same batch budget. Setting `AGENTKIT_MIGRATION_JUDGE_APP_SERVER=0` pins that scripted judge for the whole run instead of writing requests. The request declares how long the runner will listen, and the turn is sized to answer inside that window, so a batch that is too slow comes back as an answered failure instead of a verdict nobody reads. Two defects came out of the first real runs and are fixed here: the turn is now bounded by wall-clock time instead of the app-server's inactivity window, and a transport failure is answered with an error envelope instead of leaving the runner waiting. The `codex exec` fallback also uses the budget left after the channel rather than the budget the batch started with. `run_tool_turn` carries the shared one-turn contract for both the route analysis and the judge, and the analysis now uses it too. Verified with a real migration-evaluation run (Dify workflow source, Dev Sandbox): the channel delivered the verdict in 285.9s, the runner accepted it without falling back, and the report matches the legacy shape field for field. --- frontend/README.md | 19 +- frontend/server/migration/app_server.py | 49 +- frontend/server/migration/codex_tool_turn.py | 129 ++++ .../migration/evaluation/judge_app_server.py | 271 ++++++++ .../migration/evaluation/judge_channel.py | 121 ++++ .../migration/evaluation/judge_driver.py | 432 ++++++++++++ .../server/migration/evaluation/runner.py | 268 +++++++- .../server/migration/evaluation/service.py | 46 ++ .../migration_evaluation/test_judge_driver.py | 637 ++++++++++++++++++ .../migration_evaluation/test_runner.py | 594 +++++++++++++++- .../migration_evaluation/test_service.py | 63 +- .../test_migration_codex_tool_turn.py | 228 +++++++ veadk/cli/cli_frontend.py | 8 + 13 files changed, 2821 insertions(+), 44 deletions(-) create mode 100644 frontend/server/migration/codex_tool_turn.py create mode 100644 frontend/server/migration/evaluation/judge_app_server.py create mode 100644 frontend/server/migration/evaluation/judge_channel.py create mode 100644 frontend/server/migration/evaluation/judge_driver.py create mode 100644 tests/frontend/server/migration_evaluation/test_judge_driver.py create mode 100644 tests/frontend/test_migration_codex_tool_turn.py diff --git a/frontend/README.md b/frontend/README.md index 15881ace9..e0e51e4b3 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -600,9 +600,22 @@ See [deployment and operation](service/studio_release_notifier/README.md). the framework, entry point, and open questions. Structured frameworks run the preinstalled `ak migrate`; Dify and Any projects run `ak migrate --execution in-place` with Codex in the same Session. Evaluation - deploys a temporary Runtime, checkpoints per-case execution as JSONL, judges - batches in one fresh resumable Codex thread, and always reconciles Runtime - cleanup before completing or cancelling. Reports show 0–100 display scores, + deploys a temporary Runtime, checkpoints per-case execution as JSONL, and + judges batches in one fresh resumable Codex thread. Analysis and judging both + deliver their contract through an app-server dynamic tool instead of parsing + free text, and the analysis turn runs on a Studio background worker so a long + analysis never waits inside the upload request. Each judged batch is a durable + request the runner writes into the Session: Studio answers it with one + app-server turn carrying the verdict, and the runner validates and caches that + verdict before the next batch. A request stays on disk until it is answered, so + a Studio restart replays the same batch; a request that cannot be answered + falls back to `codex exec` inside the same batch budget. The request declares + how long the runner will listen, and the turn is sized to answer inside that + window, so even a judge batch that is too slow comes back as an answered + failure rather than a timeout. Setting + `AGENTKIT_MIGRATION_JUDGE_APP_SERVER=0` pins that scripted judge for the + whole run instead of writing judge requests at all. Cleanup is always + reconciled before completing or cancelling. Reports show 0–100 display scores, execution success, evidence coverage, N/A counts, low-scoring and failed cases, versions, evidence severity, and cleanup status without a pass/fail verdict. Each raw judge score is rounded half up to a 0–100 integer before diff --git a/frontend/server/migration/app_server.py b/frontend/server/migration/app_server.py index 1c99be33c..87c71290c 100644 --- a/frontend/server/migration/app_server.py +++ b/frontend/server/migration/app_server.py @@ -26,12 +26,9 @@ from collections.abc import Callable import os -from veadk.cli.codex_app_server import ( - CodexAppServerError, - CodexAppServerSession, - CodexDynamicToolResult, -) +from veadk.cli.codex_app_server import CodexDynamicToolResult +from .codex_tool_turn import ToolTurnUnavailable, run_tool_turn from .contracts import MigrationContractError, validate_analysis_result ROUTE_TOOL_NAME = "reportRoute" @@ -104,36 +101,22 @@ async def run_route_analysis( ) -> dict[str, object] | None: """Run one analysis turn and return the validated route contract, if any.""" recorder = RouteRecorder(attempt=attempt, input_sha256=input_sha256) - session = CodexAppServerSession(endpoint) - session.cwd = cwd - if model: - session.model = model - session.register_dynamic_tool( - ROUTE_TOOL_NAME, - ROUTE_TOOL_DESCRIPTION, - schema, - recorder.submit, - ) - try: - await session.connect() - except CodexAppServerError as error: - raise MigrationAnalysisUnavailable(str(error)) from error try: - async for event in session.stream_turn( - prompt, + await run_tool_turn( + endpoint=endpoint, + prompt=prompt, + cwd=cwd, + tool_name=ROUTE_TOOL_NAME, + tool_description=ROUTE_TOOL_DESCRIPTION, + tool_schema=schema, + handler=recorder.submit, + has_result=lambda: recorder.result is not None, + model=model, timeout_seconds=timeout_seconds, - ): - if event_sink is not None: - event_sink(event) - if recorder.result is not None: - # 结果已经到手：终止本轮，避免继续消耗 token 和沙箱时间。 - await session.interrupt() - break - except CodexAppServerError as error: - if recorder.result is None: - raise MigrationAnalysisUnavailable(str(error)) from error - finally: - await session.close() + event_sink=event_sink, + ) + except ToolTurnUnavailable as error: + raise MigrationAnalysisUnavailable(str(error)) from error return recorder.result diff --git a/frontend/server/migration/codex_tool_turn.py b/frontend/server/migration/codex_tool_turn.py new file mode 100644 index 000000000..1dac0811f --- /dev/null +++ b/frontend/server/migration/codex_tool_turn.py @@ -0,0 +1,129 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One Codex app-server turn that reports its result through a dynamic tool. + +Studio drives every structured Codex turn the same way: a dynamic tool registered on +``thread/start`` carries the result as typed JSON-RPC arguments, a rejected call returns +``success: false`` so the same turn can correct itself, and the turn ends as soon as the +contract has been delivered. Callers own the contract: they pass the tool spec, the +validator, and the check that says the result has arrived. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +from veadk.cli.codex_app_server import ( + CodexAppServerError, + CodexAppServerSession, + CodexDynamicToolResult, +) + +ToolHandler = Callable[[dict[str, object]], CodexDynamicToolResult] + +__all__ = [ + "ToolHandler", + "ToolTurnDeadlineExceeded", + "ToolTurnUnavailable", + "run_tool_turn", +] + + +class ToolTurnUnavailable(RuntimeError): + """The Sandbox app-server could not start or finish the tool turn.""" + + +class ToolTurnDeadlineExceeded(ToolTurnUnavailable): + """The turn kept making progress but ran past the caller's wall-clock window. + + Callers treat this differently from a transport failure: the app-server works, + the batch is simply too slow for the window that was granted. + """ + + +async def run_tool_turn( + *, + endpoint: str, + prompt: str, + cwd: str, + tool_name: str, + tool_description: str, + tool_schema: dict[str, object], + handler: ToolHandler, + has_result: Callable[[], bool], + thread_id: str = "", + model: str = "", + timeout_seconds: float, + event_sink: Callable[[object], None] | None = None, +) -> str: + """Run one turn and return the thread id that carried it. + + ``thread_id`` resumes a durable thread instead of starting a new one; the + app-server restores that thread's dynamic tools, so the same contract keeps + arriving. ``timeout_seconds`` is a wall-clock budget, not just the app-server's + inactivity window: callers that must answer inside a caller-owned window cannot + rely on a turn that keeps making progress, so this interrupts it at the deadline. + Any protocol or transport failure becomes ``ToolTurnUnavailable`` so the caller can + fall back, unless the result already arrived. + """ + session = CodexAppServerSession(endpoint) + session.cwd = cwd + if model: + session.model = model + session.register_dynamic_tool( + tool_name, + tool_description, + tool_schema, + handler, + ) + used_thread = thread_id + try: + try: + if thread_id: + await session.attach_thread(thread_id) + else: + await session.connect() + except CodexAppServerError as error: + raise ToolTurnUnavailable(str(error)) from error + deadline = asyncio.get_running_loop().time() + timeout_seconds + try: + async for event in session.stream_turn( + prompt, + timeout_seconds=timeout_seconds, + ): + if event_sink is not None: + event_sink(event) + if has_result(): + # 结果已经到手：终止本轮，避免继续消耗 token 和沙箱时间。 + await session.interrupt() + break + if asyncio.get_running_loop().time() >= deadline: + # 回合一直在产生进度，但已经超出调用方的窗口：主动收尾。 + await session.interrupt() + raise ToolTurnDeadlineExceeded("Codex 回合超出时间预算。") + except TimeoutError as error: + # app-server 客户端自己的空闲超时：同样是「没在窗口内交付」。 + if not has_result(): + raise ToolTurnDeadlineExceeded( + str(error) or "Codex 回合超时。" + ) from error + except CodexAppServerError as error: + if not has_result(): + raise ToolTurnUnavailable(str(error)) from error + used_thread = session.thread_id or thread_id + finally: + await session.close() + return used_thread diff --git a/frontend/server/migration/evaluation/judge_app_server.py b/frontend/server/migration/evaluation/judge_app_server.py new file mode 100644 index 000000000..aae659b79 --- /dev/null +++ b/frontend/server/migration/evaluation/judge_app_server.py @@ -0,0 +1,271 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Judge one evaluation batch through the Sandbox app-server and its dynamic tool. + +The legacy judge ran ``codex exec --output-schema`` and parsed the last agent message. +This contract carries the same verdicts as typed ``reportEvaluation`` arguments, so a +progress note or a fenced code block can no longer be mistaken for the batch result, and +a rejected batch is answered inside the same turn so Codex can correct itself. +""" + +from __future__ import annotations + +from veadk.cli.codex_app_server import CodexDynamicToolResult + +from ..codex_tool_turn import ToolTurnUnavailable, run_tool_turn +from .judge_channel import JUDGE_TOOL_NAME +from .runner import judge_schema + +JUDGE_TOOL_DESCRIPTION = ( + "提交本批迁移效果评测的判定结果。必须一次性提交全部用例与全部维度，" + "参数严格遵循给定的 JSON Schema；被拒绝时按返回的错误修正后重新调用。" +) + +EVIDENCE_SOURCES = frozenset( + { + "user_reference", + "user_criteria", + "source_contract", + "observed_output", + "runtime_observation", + "deterministic_assertion", + } +) +SEVERITIES = frozenset({"none", "low", "medium", "high", "critical", "unknown"}) +_WORKFLOW_DIMENSION = "workflow_tool_fidelity" +_MAX_EVIDENCE_ITEMS = 20 +_MAX_REASON_CHARS = 4 * 1024 +_MAX_EVIDENCE_CHARS = 2 * 1024 + + +class JudgeContractError(ValueError): + """One ``reportEvaluation`` payload did not satisfy the judging contract.""" + + +class JudgeTurnUnavailable(ToolTurnUnavailable): + """The Sandbox app-server could not deliver a judge batch. + + The turn ran but produced nothing the runner can validate: no verdict, or a verdict + that is not a case list. It is a ``ToolTurnUnavailable`` so the driver answers every + request that never delivered with an error envelope, whatever went wrong. + """ + + +def _required_text(value: object, field: str) -> str: + if not isinstance(value, str): + raise JudgeContractError(f"{field} 必须是字符串") + return value + + +def _bounded_text(value: object, field: str, limit: int) -> str: + text = _required_text(value, field).strip() + if not text: + raise JudgeContractError(f"{field} 不能为空") + return text[:limit] + + +def validate_judge_cases( + arguments: dict[str, object], + *, + case_context: list[dict[str, object]], + dimensions: list[str], +) -> dict[str, object]: + """Validate one batch verdict against the cases and dimensions it must cover. + + The runner repeats these checks before caching the batch, so a rejection here is + advice to Codex rather than the last line of defence. + """ + if not dimensions: + raise JudgeContractError("评测维度不能为空") + cases = arguments.get("cases") + if not isinstance(cases, list) or len(cases) != len(case_context): + raise JudgeContractError(f"cases 必须恰好包含 {len(case_context)} 个用例") + expected_ids = [str(item.get("case_id") or "") for item in case_context] + returned_ids = [ + item.get("case_id") if isinstance(item, dict) else None for item in cases + ] + if returned_ids != expected_ids: + raise JudgeContractError( + "cases 必须按给定顺序包含全部用例：" + "、".join(expected_ids) + ) + for index, (case, context) in enumerate(zip(cases, case_context)): + assert isinstance(case, dict) + results = case.get("dimensions") + if not isinstance(results, list) or len(results) != len(dimensions): + raise JudgeContractError( + f"cases[{index}].dimensions 必须恰好包含 {len(dimensions)} 个维度" + ) + returned_dimension_ids = [ + item.get("id") if isinstance(item, dict) else None for item in results + ] + if returned_dimension_ids != dimensions: + raise JudgeContractError( + f"cases[{index}].dimensions 必须按给定顺序包含全部维度：" + + "、".join(dimensions) + ) + failed_execution = str(context.get("state") or "") == "failed" + for dimension, result in zip(dimensions, results): + assert isinstance(result, dict) + field = f"cases[{index}].dimensions[{dimension}]" + score = result.get("score") + if score is not None and ( + isinstance(score, bool) + or not isinstance(score, (int, float)) + or not 0 <= score <= 1 + ): + raise JudgeContractError( + f"{field}.score 必须在 0 到 1 之间，证据不足时必须为 null" + ) + severity = result.get("severity") + if severity not in SEVERITIES: + raise JudgeContractError( + f"{field}.severity 只能是 " + "、".join(sorted(SEVERITIES)) + ) + if (score is None) is not (severity == "unknown"): + raise JudgeContractError( + f"{field} 的 score 与 severity 必须同时表示 N/A：" + "score 为 null 时 severity 必须是 unknown，反之亦然" + ) + result["reason"] = _bounded_text( + result.get("reason"), f"{field}.reason", _MAX_REASON_CHARS + ) + evidence = result.get("evidence") + if not isinstance(evidence, list) or any( + not isinstance(entry, str) for entry in evidence + ): + raise JudgeContractError(f"{field}.evidence 必须是字符串数组") + result["evidence"] = [ + entry.strip()[:_MAX_EVIDENCE_CHARS] + for entry in evidence[:_MAX_EVIDENCE_ITEMS] + if entry.strip() + ] + sources = result.get("evidence_sources") + if ( + not isinstance(sources, list) + or any(not isinstance(source, str) for source in sources) + or len(sources) != len(set(sources)) + or any(source not in EVIDENCE_SOURCES for source in sources) + ): + raise JudgeContractError( + f"{field}.evidence_sources 只能不重复地使用 " + + "、".join(sorted(EVIDENCE_SOURCES)) + ) + if failed_execution and score is not None: + raise JudgeContractError( + f"{field} 的执行用例失败，全部维度必须为 N/A（score 为 null、" + "severity 为 unknown）" + ) + if ( + dimension == _WORKFLOW_DIMENSION + and score is not None + and context.get("contract") is not True + and context.get("criteria") is not True + and context.get("runtime_observation") is not True + ): + raise JudgeContractError( + f"{field} 缺少 Runtime 原始可观察数据、用户标准和源行为契约，" + "必须为 N/A" + ) + return {"cases": cases} + + +class JudgeRecorder: + """Validate and retain the first judge batch delivered by a dynamic tool call. + + ``result`` holds the validated tool arguments, mirroring ``RouteRecorder``; the + batch verdict itself is the ``cases`` list inside them. + """ + + def __init__( + self, + *, + case_context: list[dict[str, object]], + dimensions: list[str], + ) -> None: + self.case_context = case_context + self.dimensions = dimensions + self.result: dict[str, object] | None = None + self.rejections: list[str] = [] + + def submit(self, arguments: dict[str, object]) -> CodexDynamicToolResult: + try: + validated = validate_judge_cases( + arguments, + case_context=self.case_context, + dimensions=self.dimensions, + ) + except JudgeContractError as error: + self.rejections.append(str(error)) + return CodexDynamicToolResult( + False, + f"判定结果不符合协议（{error}）。请修正后重新调用 {JUDGE_TOOL_NAME}。", + ) + if self.result is None: + self.result = validated + return CodexDynamicToolResult( + True, + "本批判定已接收，请直接结束本轮，不要再输出其他内容。", + ) + + +async def run_judge_turn( + *, + endpoint: str, + prompt: str, + cwd: str, + case_context: list[dict[str, object]], + dimensions: list[str], + thread_id: str = "", + model: str = "", + timeout_seconds: float, +) -> tuple[list[dict[str, object]], str]: + """Judge one batch and return ``(cases, thread_id)``. + + The verdict is the ``cases`` list, not the whole tool call: the runner validates + that list against the batch it asked about, so Studio must not wrap it again. + """ + recorder = JudgeRecorder(case_context=case_context, dimensions=dimensions) + used_thread = await run_tool_turn( + endpoint=endpoint, + prompt=prompt, + cwd=cwd, + tool_name=JUDGE_TOOL_NAME, + tool_description=JUDGE_TOOL_DESCRIPTION, + tool_schema=judge_schema(), + handler=recorder.submit, + has_result=lambda: recorder.result is not None, + thread_id=thread_id, + model=model, + timeout_seconds=timeout_seconds, + ) + if recorder.result is None: + raise JudgeTurnUnavailable("裁判回合没有提交判定结果。") + cases = recorder.result.get("cases") + if not isinstance(cases, list): + raise JudgeTurnUnavailable("裁判回合没有提交可用的判定结果。") + return cases, used_thread + + +__all__ = [ + "EVIDENCE_SOURCES", + "JUDGE_TOOL_DESCRIPTION", + "JUDGE_TOOL_NAME", + "JudgeContractError", + "JudgeRecorder", + "JudgeTurnUnavailable", + "SEVERITIES", + "run_judge_turn", + "validate_judge_cases", +] diff --git a/frontend/server/migration/evaluation/judge_channel.py b/frontend/server/migration/evaluation/judge_channel.py new file mode 100644 index 000000000..ebdf146a9 --- /dev/null +++ b/frontend/server/migration/evaluation/judge_channel.py @@ -0,0 +1,121 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Durable request/response channel between the in-Sandbox judge and Studio. + +The evaluation runner stays the orchestrator: it owns batching, the thread record, the +batch cache, and the report. Studio owns the one capability the Sandbox cannot host - +a Codex app-server turn that delivers its verdict through a dynamic tool - so the two +sides meet on a single-slot file channel inside the Sandbox: + +* the runner writes ``request.json`` for the batch it is judging and waits for the + ``response.json`` that names the same request id; +* Studio reads the request, runs one app-server turn, and writes that response. + +The channel is a pure contract: it holds the file names, the enable switch, and the +dynamic tool name so both sides cannot drift, and it imports nothing from the rest of +the migration package. + +Request:: + + {"schema_version": 1, "request_id": "batch-001-010-attempt-1", + "batch_start": 0, "batch_end": 10, "case_ids": ["case-1"], + "dimensions": ["semantic_fidelity"], "prompt_version": 4, + "thread_id": "", "created_at": "2025-01-01T00:00:00Z", "prompt": "...", + "budget_seconds": 360, + "case_context": [{"case_id": "case-1", "state": "succeeded", + "criteria": True, "contract": False, + "runtime_observation": True}]} + +Response on success:: + + {"schema_version": 1, "request_id": "batch-001-010-attempt-1", "ok": True, + "thread_id": "thread-1", "cases": [...], "created_at": "..."} + +Response on failure, so the runner falls back to ``codex exec`` instead of waiting for +a turn that will never arrive:: + + {"schema_version": 1, "request_id": "batch-001-010-attempt-1", "ok": False, + "thread_id": "", "error": {"code": "judge_turn_unavailable", + "message": "..."}} + +``budget_seconds`` is how long the runner will wait for the response. It is the only +deadline in the protocol: Studio sizes its turn to answer inside that window, so a slow +turn reaches the runner as ``ok: false`` while the runner is still listening instead of +as a verdict nobody reads. + +Both sides key on the request id, so a response that arrives late, out of order, or +twice is either ignored or replayed for the same batch verdict - never applied to a +different batch. +""" + +from __future__ import annotations + +import os + +JUDGE_CHANNEL_SCHEMA_VERSION = 1 +JUDGE_CHANNEL_DIRECTORY = "judge" +JUDGE_REQUEST_NAME = "request.json" +JUDGE_RESPONSE_NAME = "response.json" + +JUDGE_TOOL_NAME = "reportEvaluation" +JUDGE_APP_SERVER_ENV = "AGENTKIT_MIGRATION_JUDGE_APP_SERVER" +_DISABLED_VALUES = {"0", "false", "no", "off"} + +EVALUATION_RESULTS_DIRECTORY = "results" +EVALUATION_ATTEMPT_DIRECTORY_PREFIX = "attempt-" + +__all__ = [ + "EVALUATION_ATTEMPT_DIRECTORY_PREFIX", + "EVALUATION_RESULTS_DIRECTORY", + "JUDGE_APP_SERVER_ENV", + "JUDGE_CHANNEL_DIRECTORY", + "JUDGE_CHANNEL_SCHEMA_VERSION", + "JUDGE_REQUEST_NAME", + "JUDGE_RESPONSE_NAME", + "JUDGE_TOOL_NAME", + "evaluation_result_root", + "judge_app_server_enabled", + "judge_channel_paths", +] + + +def judge_app_server_enabled() -> bool: + """Whether batch judging runs through the app-server instead of ``codex exec``. + + The runner still owns batching, the thread record, the batch cache, and the report; + Studio only answers the judge request. Set ``AGENTKIT_MIGRATION_JUDGE_APP_SERVER=0`` + to pin the scripted judge, which the runner also falls back to whenever this path + cannot deliver a batch. + """ + return os.getenv(JUDGE_APP_SERVER_ENV, "").strip().lower() not in _DISABLED_VALUES + + +def evaluation_result_root(evaluation_root: str, attempt: int) -> str: + """Return the directory that holds one evaluation attempt's durable results.""" + return ( + f"{evaluation_root.rstrip('/')}/{EVALUATION_RESULTS_DIRECTORY}/" + f"{EVALUATION_ATTEMPT_DIRECTORY_PREFIX}{attempt}" + ) + + +def judge_channel_paths(evaluation_root: str, attempt: int) -> tuple[str, str]: + """Return the ``(request_path, response_path)`` of one evaluation attempt.""" + base = ( + f"{evaluation_result_root(evaluation_root, attempt)}/{JUDGE_CHANNEL_DIRECTORY}" + ) + return ( + f"{base}/{JUDGE_REQUEST_NAME}", + f"{base}/{JUDGE_RESPONSE_NAME}", + ) diff --git a/frontend/server/migration/evaluation/judge_driver.py b/frontend/server/migration/evaluation/judge_driver.py new file mode 100644 index 000000000..b082c4acc --- /dev/null +++ b/frontend/server/migration/evaluation/judge_driver.py @@ -0,0 +1,432 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Studio side of the judge channel: one app-server turn per Sandbox request. + +The runner keeps owning batching, the thread record, the batch cache, and the report; +this driver only answers the request the runner writes. It is driven by the evaluation +watcher, so every property that matters comes from being replayable: + +* the request stays on disk until it has an answer, so a Studio restart re-runs the + same turn instead of losing the batch; +* one turn per request id runs at a time, and the answer is written once, so a slow + turn is never started twice; +* a turn that cannot deliver is answered with an error envelope, which sends the + runner straight back to ``codex exec`` instead of leaving it waiting. + +The request declares how long the runner will wait (``budget_seconds``); the turn is +sized to answer inside that window, so the runner always reads an answer - a verdict or +a reason - instead of timing out on work that was still running. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from dataclasses import dataclass +import json +import threading +import time + +from veadk.utils.logger import get_logger + +from ..gateway import ( + MigrationGateway, + MigrationGatewayError, + MigrationRemoteFileNotFound, + MigrationSandboxSession, +) +from ..codex_tool_turn import ( + ToolTurnDeadlineExceeded, + ToolTurnUnavailable, +) +from .judge_app_server import run_judge_turn +from .judge_channel import ( + JUDGE_CHANNEL_SCHEMA_VERSION, + judge_app_server_enabled, + judge_channel_paths, +) + +logger = get_logger(__name__) + +# 单条请求可能内嵌整批用例与 Runtime 观测，读取上限与评测报告保持一致。 +JUDGE_REQUEST_MAX_BYTES = 16 * 1024 * 1024 +# 单次回合的上限（不依赖 runner 给多少预算时的兜底值）。 +# 实测一次成功判定约 130s，慢回合会超过 240s，因此这里留到 300s。 +JUDGE_TURN_TIMEOUT_SECONDS = 300.0 +# 续跑既有线程的预算；剩余预算留给「线程已失效，改用新线程」的第二次尝试。 +JUDGE_RESUME_TIMEOUT_SECONDS = 90.0 +# 从 runner 给的窗口里预留给「取件 + 落盘 + runner 轮询」的时间。 +# runner 必须早于它自己的等待上限拿到信封，否则会先超时降级。 +JUDGE_TURN_RESERVE_SECONDS = 30.0 +_CASE_CONTEXT_FLAGS = ("criteria", "contract", "runtime_observation") + +__all__ = [ + "JUDGE_REQUEST_MAX_BYTES", + "JUDGE_RESUME_TIMEOUT_SECONDS", + "JUDGE_TURN_RESERVE_SECONDS", + "JUDGE_TURN_TIMEOUT_SECONDS", + "JudgeRequest", + "JudgeRequestError", + "SandboxJudgeDriver", + "answer_judge_request", + "judge_turn_budget", + "parse_judge_request", +] + + +class JudgeRequestError(RuntimeError): + """One Sandbox judge request is not a valid channel message.""" + + +@dataclass(frozen=True) +class JudgeRequest: + """One batch the Sandbox is asking Studio to judge.""" + + request_id: str + prompt: str + case_context: tuple[dict[str, object], ...] + dimensions: tuple[str, ...] + thread_id: str + # runner 授予的应答窗口（秒）；0 表示请求方没有声明，用本地上限兜底。 + budget_seconds: float = 0.0 + + @property + def case_ids(self) -> tuple[str, ...]: + return tuple(str(entry["case_id"]) for entry in self.case_context) + + +def parse_judge_request(value: object) -> JudgeRequest: + """Parse and bound one ``request.json`` payload.""" + if ( + not isinstance(value, dict) + or value.get("schema_version") != JUDGE_CHANNEL_SCHEMA_VERSION + ): + raise JudgeRequestError("评测裁判请求的协议版本不匹配") + request_id = value.get("request_id") + if not isinstance(request_id, str) or not request_id or len(request_id) > 128: + raise JudgeRequestError("评测裁判请求缺少有效的 request_id") + prompt = value.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise JudgeRequestError("评测裁判请求缺少提示词") + dimensions = value.get("dimensions") + if ( + not isinstance(dimensions, list) + or not dimensions + or len(set(dimensions)) != len(dimensions) + or any(not isinstance(item, str) or not item for item in dimensions) + ): + raise JudgeRequestError("评测裁判请求的维度无效") + entries = value.get("case_context") + if not isinstance(entries, list) or not entries or len(entries) > 64: + raise JudgeRequestError("评测裁判请求的用例上下文无效") + case_context: list[dict[str, object]] = [] + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise JudgeRequestError(f"评测裁判请求的用例上下文 {index} 无效") + case_id = entry.get("case_id") + if not isinstance(case_id, str) or not case_id: + raise JudgeRequestError(f"评测裁判请求的用例上下文 {index} 缺少 case_id") + state = entry.get("state") + context: dict[str, object] = { + "case_id": case_id, + "state": state if isinstance(state, str) else "", + } + for flag in _CASE_CONTEXT_FLAGS: + context[flag] = entry.get(flag) is True + case_context.append(context) + thread_id = value.get("thread_id") + if thread_id is None: + thread_id = "" + if not isinstance(thread_id, str) or len(thread_id) > 256: + raise JudgeRequestError("评测裁判请求的线程标识无效") + budget = value.get("budget_seconds") + if budget is None: + budget_seconds = 0.0 + elif ( + isinstance(budget, bool) + or not isinstance(budget, (int, float)) + or budget <= 0 + or budget > 24 * 3600 + ): + raise JudgeRequestError("评测裁判请求的时间预算无效") + else: + budget_seconds = float(budget) + return JudgeRequest( + request_id=request_id, + prompt=prompt, + case_context=tuple(case_context), + dimensions=tuple(dimensions), + thread_id=thread_id, + budget_seconds=budget_seconds, + ) + + +def judge_turn_budget(request: JudgeRequest) -> float: + """How long this turn may run before it must answer the runner. + + The runner owns the deadline: it declares how long it will wait, and the turn has + to finish early enough for its answer to reach the runner before that window + closes. Without a declared window the local ceiling applies. + """ + if request.budget_seconds <= 0: + return JUDGE_TURN_TIMEOUT_SECONDS + return min( + JUDGE_TURN_TIMEOUT_SECONDS, + request.budget_seconds - JUDGE_TURN_RESERVE_SECONDS, + ) + + +def _judge_response( + request: JudgeRequest, + *, + cases: dict[str, object] | None = None, + thread_id: str = "", + error: dict[str, str] | None = None, +) -> dict[str, object]: + value: dict[str, object] = { + "schema_version": JUDGE_CHANNEL_SCHEMA_VERSION, + "request_id": request.request_id, + "ok": error is None, + "thread_id": thread_id, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + if error is None: + value["cases"] = cases + else: + value["error"] = error + return value + + +def answer_judge_request( + request: JudgeRequest, + *, + endpoint: str, + cwd: str, + model: str = "", + timeout_seconds: float | None = None, +) -> dict[str, object]: + """Run the app-server turn for one batch and return the response envelope. + + A resumed thread that can no longer be attached is retried once on a fresh thread: + losing the judge thread costs continuity, while failing the batch costs the report. + Every step of that plan shares one deadline, so the envelope is written while the + runner is still waiting for it. + """ + if timeout_seconds is None: + timeout_seconds = judge_turn_budget(request) + started = time.monotonic() + deadline = started + timeout_seconds + plan: list[tuple[str, float]] = [] + if request.thread_id: + plan.append((request.thread_id, JUDGE_RESUME_TIMEOUT_SECONDS)) + plan.append(("", 0.0)) + failure: Exception | None = None + for thread_id, resume_budget in plan: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + if resume_budget > 0: + remaining = min(resume_budget, remaining) + try: + cases, used_thread = asyncio.run( + run_judge_turn( + endpoint=endpoint, + prompt=request.prompt, + cwd=cwd, + case_context=[dict(entry) for entry in request.case_context], + dimensions=list(request.dimensions), + thread_id=thread_id, + model=model, + timeout_seconds=remaining, + ) + ) + except ToolTurnUnavailable as error: + failure = error + continue + except Exception as error: # noqa: BLE001 - 任何回合故障都必须变成信封 + # 不认识的问题也必须有答案，否则 runner 会一直等到自己的窗口结束。 + logger.exception( + "Studio judge turn raised request_id=%s error_type=%s", + request.request_id, + type(error).__name__, + ) + failure = error + continue + logger.info( + "Studio judge turn delivered request_id=%s elapsed=%.1fs thread_id=%s", + request.request_id, + time.monotonic() - started, + used_thread, + ) + return _judge_response(request, cases=cases, thread_id=used_thread) + ran_out_of_time = failure is None or isinstance(failure, ToolTurnDeadlineExceeded) + detail = str(failure) if failure is not None else "评测裁判回合超出时间预算" + logger.warning( + "Studio judge turn unavailable request_id=%s error_type=%s budget=%.0fs " + "elapsed=%.1fs", + request.request_id, + type(failure).__name__ if failure is not None else "timeout", + timeout_seconds, + time.monotonic() - started, + ) + return _judge_response( + request, + error={ + "code": ( + "judge_turn_timeout" if ran_out_of_time else "judge_turn_unavailable" + ), + "message": detail[:512], + }, + ) + + +class SandboxJudgeDriver: + """Answer pending judge requests for the evaluation attempts of one Studio run.""" + + def __init__( + self, + gateway: MigrationGateway, + *, + cwd: str, + model: str = "", + enabled: Callable[[], bool] = judge_app_server_enabled, + ) -> None: + self._gateway = gateway + self._cwd = cwd + self._model = model + self._enabled = enabled + self._turns: dict[tuple[str, int], threading.Thread] = {} + self._answered: dict[tuple[str, int], set[str]] = {} + + def drive( + self, + session: MigrationSandboxSession, + *, + evaluation_root: str, + attempt: int, + ) -> None: + """Answer the pending request of this attempt, if there is an unanswered one. + + Cheap to call on every watcher tick: it reads one small file and returns unless + the runner is waiting for a batch Studio has not answered yet. + """ + if attempt < 1 or not self._enabled(): + return + key = (session.session_id, attempt) + running = self._turns.get(key) + if running is not None and running.is_alive(): + return + request_path, _ = judge_channel_paths(evaluation_root, attempt) + try: + content = self._gateway.get_file( + session, + request_path, + max_bytes=JUDGE_REQUEST_MAX_BYTES, + ) + except MigrationRemoteFileNotFound: + return + except MigrationGatewayError as error: + logger.warning( + "Studio judge channel read failed task_id=%s attempt=%s code=%s", + session.task_id, + attempt, + error.code, + ) + return + try: + request = parse_judge_request(json.loads(content)) + except (UnicodeDecodeError, ValueError, JudgeRequestError) as error: + logger.warning( + "Studio judge channel request rejected task_id=%s attempt=%s " + "error_type=%s", + session.task_id, + attempt, + type(error).__name__, + ) + return + if request.request_id in self._answered.get(key, frozenset()): + return + logger.info( + "Studio judge channel turn started task_id=%s attempt=%s request_id=%s " + "budget=%.0fs thread=%s", + session.task_id, + attempt, + request.request_id, + judge_turn_budget(request), + "resumed" if request.thread_id else "new", + ) + worker = threading.Thread( + target=self._answer, + args=(session, attempt, evaluation_root, key, request), + name=f"migration-judge-{attempt}", + daemon=True, + ) + self._turns[key] = worker + try: + worker.start() + except Exception: # noqa: BLE001 - a failed start must not break the tick + self._turns.pop(key, None) + logger.exception( + "Studio judge worker could not start task_id=%s attempt=%s", + session.task_id, + attempt, + ) + + def _answer( + self, + session: MigrationSandboxSession, + attempt: int, + evaluation_root: str, + key: tuple[str, int], + request: JudgeRequest, + ) -> None: + try: + response = answer_judge_request( + request, + endpoint=session.endpoint, + cwd=self._cwd, + model=self._model, + ) + _, response_path = judge_channel_paths(evaluation_root, attempt) + self._gateway.put_file( + session, + response_path, + json.dumps( + response, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8"), + media_type="application/json", + ) + self._answered.setdefault(key, set()).add(request.request_id) + logger.info( + "Studio judge channel answered task_id=%s attempt=%s request_id=%s " + "ok=%s", + session.task_id, + attempt, + request.request_id, + str(response.get("ok")).lower(), + ) + except Exception: # noqa: BLE001 - the watcher tick must survive this + logger.exception( + "Studio judge channel answer failed task_id=%s attempt=%s " + "request_id=%s", + session.task_id, + attempt, + request.request_id, + ) + finally: + if self._turns.get(key) is threading.current_thread(): + self._turns.pop(key, None) diff --git a/frontend/server/migration/evaluation/runner.py b/frontend/server/migration/evaluation/runner.py index 561fea4e2..18d67e418 100644 --- a/frontend/server/migration/evaluation/runner.py +++ b/frontend/server/migration/evaluation/runner.py @@ -34,6 +34,12 @@ MigrationSandboxSession, ) from ..service import MIGRATION_ROOT, MigrationError +from .judge_channel import ( + JUDGE_TOOL_NAME, + evaluation_result_root, + judge_app_server_enabled, + judge_channel_paths, +) from .service import ( EVALUATION_DATASET_PATH, EVALUATION_REPORT_PATH, @@ -51,6 +57,8 @@ _RUNNER_PATH = f"{EVALUATION_ROOT}/assets/evaluation_runner.py" _JUDGE_SCHEMA_PATH = f"{EVALUATION_ROOT}/assets/judge-schema.json" +# 评委回合也在迁移产物目录里执行，与旧版 codex exec 裁判的 --cd 保持一致。 +EVALUATION_PROJECT_PATH = f"{MIGRATION_ROOT}/output/veadk" _PROJECT_CONFIG_PATHS = ( f"{MIGRATION_ROOT}/output/veadk/agentkit.yaml", f"{MIGRATION_ROOT}/output/veadk/.agentkit/agentkit.yaml", @@ -250,8 +258,18 @@ def runner_source() -> str: RUNTIME_OBSERVATION_LIMIT = 16 * 1024 RAW_LIMIT = 16 * 1024 * 1024 INVOKE_TIMEOUT = 120 - JUDGE_TIMEOUT = 300 - JUDGE_PROMPT_VERSION = 3 + # 单批总预算：先请 Studio 判定，失败时剩下的时间留给 codex exec 兜底。 + # 必须盖住「通道等待上限 + 一次 codex exec（实测约 140s）」。 + JUDGE_TIMEOUT = 540 + # 等待 Studio 应答的上限。必须大于 Studio 回合上限加上它的落盘预留 + # （judge_driver.JUDGE_TURN_TIMEOUT_SECONDS + JUDGE_TURN_RESERVE_SECONDS）， + # 否则 runner 会先超时，Studio 递回来的信封就白写了。 + JUDGE_CHANNEL_TIMEOUT = 360 + JUDGE_CHANNEL_POLL_SECONDS = 3 + JUDGE_CHANNEL_RESPONSE_LIMIT = 16 * 1024 * 1024 + JUDGE_CHANNEL_SCHEMA_VERSION = 1 + JUDGE_CHANNEL_DISABLED_RUNS = set() + JUDGE_PROMPT_VERSION = 4 EXECUTION_RESULT_LIMIT = 12 * 1024 * 1024 EVIDENCE_SOURCES = { "user_reference", @@ -1039,6 +1057,187 @@ def codex_events(events): return thread_id, message + def judge_channel(config): + channel = config.get("judge_channel") + if not isinstance(channel, dict): + return None + if config.get("batch_root_path") in JUDGE_CHANNEL_DISABLED_RUNS: + return None + request_path = channel.get("request_path") + response_path = channel.get("response_path") + tool_name = channel.get("tool_name") + if not isinstance(request_path, str) or not request_path: + return None + if not isinstance(response_path, str) or not response_path: + return None + if not isinstance(tool_name, str) or not tool_name: + return None + return request_path, response_path, tool_name + + + def disable_judge_channel(config): + # 通道交付失败后，本轮剩余批次直接使用 codex exec，不再逐批白等预算。 + JUDGE_CHANNEL_DISABLED_RUNS.add(config.get("batch_root_path")) + + + def judge_request_id(batch_start, cases, judge_attempt): + return "batch-{:03d}-{:03d}-attempt-{}".format( + batch_start + 1, + batch_start + len(cases), + judge_attempt, + ) + + + def judge_case_context(cases, observations, contract): + # 只描述每个用例允许裁判依据什么，不重复携带用例负载。 + context = [] + for case in cases: + observation = observations.get(case["case_id"]) + runtime_observation = ( + observation.get("runtime_observation") + if isinstance(observation, dict) + else None + ) + context.append( + { + "case_id": case["case_id"], + "state": ( + str(observation.get("state") or "") + if isinstance(observation, dict) + else "" + ), + "criteria": bool(case.get("criteria")), + "contract": contract is not None, + "runtime_observation": bool( + isinstance(runtime_observation, dict) + and str(runtime_observation.get("text") or "").strip() + ), + } + ) + return context + + + def read_judge_response(response_path, request_id): + # 返回绑定到 request_id 的响应，尚未写好时返回 None。 + # 撕裂或残留的响应与「还没写好」无法区分，两者都只是继续等待； + # 只有带同一 request_id 的完整响应才会被采用。 + target = Path(response_path) + try: + if not target.is_file(): + return None + raw = target.read_bytes() + except OSError: + return None + if len(raw) > JUDGE_CHANNEL_RESPONSE_LIMIT: + return None + try: + value = json.loads(raw) + except ValueError: + return None + if not isinstance(value, dict) or value.get("request_id") != request_id: + return None + return value + + + def await_judge_response(response_path, request_id, deadline): + while True: + response = read_judge_response(response_path, request_id) + if response is not None: + return response + if time.monotonic() >= deadline: + return None + time.sleep(JUDGE_CHANNEL_POLL_SECONDS) + + + def judge_through_channel( + config, + channel, + prompt, + batch_start, + cases, + judge_attempt, + case_context, + thread_id, + observations, + contract, + remaining, + ): + # 请 Studio 裁判这一批；返回 None 表示本批要回退到 codex exec。 + # 请求落盘即可重放：Studio 重启后会重新执行同一个回合，而不是丢掉这一批。 + request_path, response_path, _tool_name = channel + request_id = judge_request_id(batch_start, cases, judge_attempt) + try: + atomic_json( + request_path, + { + "schema_version": JUDGE_CHANNEL_SCHEMA_VERSION, + "request_id": request_id, + "batch_start": batch_start, + "batch_end": batch_start + len(cases), + "case_ids": [case["case_id"] for case in cases], + "dimensions": config["dimensions"], + "prompt_version": JUDGE_PROMPT_VERSION, + # 把 runner 的等待窗口交给 Studio：回合必须早于它给出信封。 + "budget_seconds": round( + min(remaining, JUDGE_CHANNEL_TIMEOUT), 3 + ), + "thread_id": thread_id or "", + "prompt": prompt, + "case_context": case_context, + "created_at": now(), + }, + ) + except OSError as error: + diagnostic( + config, + "judge_channel_write_failed", + error_type=type(error).__name__, + ) + return None + deadline = time.monotonic() + min(remaining, JUDGE_CHANNEL_TIMEOUT) + response = await_judge_response(response_path, request_id, deadline) + if response is None: + diagnostic(config, "judge_channel_timeout", detail=request_id) + return None + if response.get("ok") is not True: + error = response.get("error") + diagnostic( + config, + "judge_channel_unavailable", + detail=( + str(error.get("code") or "unknown") + if isinstance(error, dict) + else "unknown" + ), + ) + return None + response_thread_id = response.get("thread_id") + if ( + not isinstance(response_thread_id, str) + or not response_thread_id + or len(response_thread_id) > 256 + ): + diagnostic(config, "judge_channel_thread_invalid") + return None + try: + returned = validate_judged_cases( + config, + cases, + response.get("cases"), + observations, + contract, + ) + except (ValueError, RuntimeError) as error: + diagnostic( + config, + "judge_channel_output_rejected", + error_type=type(error).__name__, + detail=str(error), + ) + return None + return response_thread_id, returned + + def judge_binding(config): return { "schema_version": 1, @@ -1348,6 +1547,24 @@ def judge_batch(config, batch_start, cases, observations, contract, env): thread_id = load_judge_thread(config) if thread_id is None and batch_start > 0: raise RuntimeError("judge thread record is missing") + channel = judge_channel(config) + case_context = judge_case_context(cases, observations, contract) + if channel is not None: + tool_name = channel[2] + prompt = "\n".join( + [ + localized( + config, + "必须通过调用 " + + tool_name + + " 工具一次性提交本批判定结果；不要用任何其他方式输出判定结果。", + "Submit the batch verdict by calling the " + + tool_name + + " tool exactly once; do not report the verdict in any other way.", + ), + prompt, + ] + ) last_error = None deadline = time.monotonic() + JUDGE_TIMEOUT batch_number = batch_start // 10 + 1 @@ -1367,6 +1584,34 @@ def judge_batch(config, batch_start, cases, observations, contract, env): diagnostic(config, "judge_time_budget_exhausted") last_error = RuntimeError("evaluation judge time budget exhausted") break + if channel is not None: + answer = judge_through_channel( + config, + channel, + prompt, + batch_start, + cases, + judge_attempt, + case_context, + thread_id, + observations, + contract, + remaining, + ) + if answer is not None: + response_thread_id, returned = answer + if response_thread_id != thread_id: + # Studio 可能因为线程失效而换了新线程，跟随它继续对话。 + thread_id = response_thread_id + save_judge_thread(config, thread_id) + save_batch_result(config, batch_start, cases, returned) + return returned + # 通道这次没有交付：本批和本轮剩下的次数都走 codex exec 兜底。 + disable_judge_channel(config) + channel = None + # 通道已经消耗掉一段预算，兜底必须按剩余时间重算： + # 沿用进入本批时的 remaining 会让一次慢通道把本批拖过 JUDGE_TIMEOUT。 + remaining = deadline - time.monotonic() command = [ "codex", "exec", @@ -1972,7 +2217,11 @@ def start( ) -> None: config_path = f"{EVALUATION_ROOT}/control/runner-{attempt}.json" work_path = f"{EVALUATION_ROOT}/attempts/{attempt}" - result_path = f"{EVALUATION_ROOT}/results/attempt-{attempt}" + result_path = evaluation_result_root(EVALUATION_ROOT, attempt) + judge_request_path, judge_response_path = judge_channel_paths( + EVALUATION_ROOT, + attempt, + ) cloud_credential_path = self._cloud_credential_path(attempt) agentkit_config_protocol, agentkit_config = self._agentkit_config(session) access_key, secret_key, session_token, cloud_credentials = ( @@ -2034,10 +2283,20 @@ def start( "status_path": EVALUATION_STATUS_PATH, "report_path": EVALUATION_REPORT_PATH, "judge_schema_path": _JUDGE_SCHEMA_PATH, - "project_path": f"{MIGRATION_ROOT}/output/veadk", + "project_path": EVALUATION_PROJECT_PATH, "work_path": work_path, "thread_path": f"{result_path}/thread.json", "batch_root_path": f"{result_path}/batches", + # 评测裁判的通道由 Studio 决定：关闭时脚本直接使用旧版 codex exec 裁判。 + "judge_channel": ( + { + "request_path": judge_request_path, + "response_path": judge_response_path, + "tool_name": JUDGE_TOOL_NAME, + } + if judge_app_server_enabled() + else None + ), "execution_results_path": f"{result_path}/execution-results.jsonl", "diagnostic_path": f"{EVALUATION_ROOT}/diagnostics/evaluation.log", "secret_path": secret_path, @@ -2397,6 +2656,7 @@ def _expiry_epoch(session: MigrationSandboxSession) -> float: __all__ = [ + "EVALUATION_PROJECT_PATH", "SandboxMigrationEvaluationRunner", "judge_schema", "runner_source", diff --git a/frontend/server/migration/evaluation/service.py b/frontend/server/migration/evaluation/service.py index 02fad571f..03263cd30 100644 --- a/frontend/server/migration/evaluation/service.py +++ b/frontend/server/migration/evaluation/service.py @@ -196,6 +196,18 @@ def stop( ) -> None: ... +class EvaluationJudgeDriver(Protocol): + """Answers the judge requests the Sandbox runner writes for one attempt.""" + + def drive( + self, + session: MigrationSandboxSession, + *, + evaluation_root: str, + attempt: int, + ) -> None: ... + + class MigrationEvaluationService: def __init__( self, @@ -204,12 +216,14 @@ def __init__( *, repository: EvaluationAssetRepository | None, runner: EvaluationRunner | None, + judge_driver: EvaluationJudgeDriver | None = None, clock: Callable[[], float] = time.time, ) -> None: self._migration = migration self._gateway = gateway self._repository = repository self._runner = runner + self._judge_driver = judge_driver self._clock = clock @property @@ -545,6 +559,7 @@ def advance( ) return if state in _ACTIVE_EVALUATION_STATES: + self._drive_judge_channel(session, task_id, status) self._reconcile_active_runner(session, task_id, status) return if state in { @@ -1032,6 +1047,36 @@ def _persist_report( report_asset=metadata.public(), ) + def _drive_judge_channel( + self, + session: MigrationSandboxSession, + task_id: str, + status: _EvaluationStatus | None, + ) -> None: + """Let Studio answer a pending Sandbox judge request. + + A cheap read that never raises, so it runs before the exit-code check: a request + on disk means the runner is waiting for a verdict, and the turn itself runs on a + driver-owned worker instead of blocking the watcher tick. + """ + if status is None or self._judge_driver is None: + return + attempt = int(status.get("attempt") or 0) + if attempt < 1: + return + try: + self._judge_driver.drive( + session, + evaluation_root=EVALUATION_ROOT, + attempt=attempt, + ) + except Exception: # noqa: BLE001 - a channel hiccup must not stop the watcher + logger.exception( + "Migration evaluation judge channel drive failed task_id=%s attempt=%s", + task_id, + attempt, + ) + def _reconcile_active_runner( self, session: MigrationSandboxSession, @@ -1611,6 +1656,7 @@ def _json_bytes(value: object) -> bytes: "EVALUATION_DATASET_PATH", "EVALUATION_REPORT_PATH", "EVALUATION_ROOT", + "EvaluationJudgeDriver", "EVALUATION_RUNNER_DIAGNOSTICS_ROOT", "EVALUATION_SECRET_PATH", "EVALUATION_STATUS_PATH", diff --git a/tests/frontend/server/migration_evaluation/test_judge_driver.py b/tests/frontend/server/migration_evaluation/test_judge_driver.py new file mode 100644 index 000000000..63b951d29 --- /dev/null +++ b/tests/frontend/server/migration_evaluation/test_judge_driver.py @@ -0,0 +1,637 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from datetime import datetime, timezone +import threading +import time +from typing import Any + +import pytest + +from frontend.server.migration.evaluation import judge_driver as judge_driver_module +from frontend.server.migration.codex_tool_turn import ( + ToolTurnDeadlineExceeded, + ToolTurnUnavailable, +) +from frontend.server.migration.evaluation.judge_app_server import ( + JudgeTurnUnavailable, +) +from frontend.server.migration.evaluation.judge_channel import ( + JUDGE_APP_SERVER_ENV, + JUDGE_CHANNEL_SCHEMA_VERSION, + judge_app_server_enabled, + judge_channel_paths, +) +from frontend.server.migration.evaluation.judge_driver import ( + JUDGE_TURN_RESERVE_SECONDS, + JUDGE_TURN_TIMEOUT_SECONDS, + JudgeRequestError, + SandboxJudgeDriver, + answer_judge_request, + judge_turn_budget, + parse_judge_request, +) +from frontend.server.migration.gateway import ( + MigrationRemoteFileNotFound, + MigrationSandboxSession, +) + +TASK_ID = "migration-v1-" + "1" * 32 +EVALUATION_ROOT = "/migration/evaluation/v1" + + +class FakeGateway: + def __init__(self) -> None: + self.files: dict[str, bytes] = {} + self.puts: list[str] = [] + + def put_file( + self, + _session: MigrationSandboxSession, + path: str, + content: bytes, + *, + media_type: str, + ) -> None: + assert media_type + self.puts.append(path) + self.files[path] = content + + def get_file( + self, + _session: MigrationSandboxSession, + path: str, + *, + max_bytes: int, + ) -> bytes: + if path not in self.files: + raise MigrationRemoteFileNotFound(path) + content = self.files[path] + assert len(content) <= max_bytes + return content + + +def _session() -> MigrationSandboxSession: + return MigrationSandboxSession( + tool_id="tool", + session_id="session", + task_id=TASK_ID, + endpoint="https://sandbox.invalid", + region="cn-beijing", + status="Ready", + created_at="2026-09-07T09:00:00Z", + expire_at=datetime.fromtimestamp( + datetime(2026, 9, 7, 11, tzinfo=timezone.utc).timestamp() + 7200, + timezone.utc, + ) + .isoformat() + .replace("+00:00", "Z"), + owner_id="owner", + ) + + +def _request_payload(**overrides: Any) -> dict[str, Any]: + payload: dict[str, Any] = { + "schema_version": JUDGE_CHANNEL_SCHEMA_VERSION, + "request_id": "batch-001-002-attempt-1", + "batch_start": 0, + "batch_end": 2, + "case_ids": ["case-1", "case-2"], + "dimensions": ["semantic_fidelity", "output_contract"], + "prompt_version": 4, + "thread_id": "", + "prompt": "judge this batch", + "created_at": "2026-09-07T10:00:00Z", + "case_context": [ + { + "case_id": "case-1", + "state": "succeeded", + "criteria": True, + "contract": False, + "runtime_observation": True, + }, + { + "case_id": "case-2", + "state": "failed", + "criteria": False, + "contract": False, + "runtime_observation": False, + }, + ], + } + payload.update(overrides) + return payload + + +_JUDGED_CASES: list[dict[str, Any]] = [ + { + "case_id": "case-1", + "dimensions": [ + { + "id": "semantic_fidelity", + "score": 0.8, + "reason": "一致", + "evidence": ["输出证据"], + "evidence_sources": ["observed_output"], + "severity": "low", + } + ], + } +] + + +def _wait_for(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return False + + +@pytest.mark.parametrize("value", ["0", "false", "no", "off", " OFF "]) +def test_judge_channel_switch_can_pin_the_scripted_judge( + monkeypatch: pytest.MonkeyPatch, + value: str, +) -> None: + monkeypatch.setenv(JUDGE_APP_SERVER_ENV, value) + + assert judge_app_server_enabled() is False + + +@pytest.mark.parametrize("value", ["", "1", "true", "yes"]) +def test_judge_channel_is_on_by_default( + monkeypatch: pytest.MonkeyPatch, + value: str, +) -> None: + if value: + monkeypatch.setenv(JUDGE_APP_SERVER_ENV, value) + else: + monkeypatch.delenv(JUDGE_APP_SERVER_ENV, raising=False) + + assert judge_app_server_enabled() is True + + +def test_judge_channel_paths_are_attempt_scoped() -> None: + request, response = judge_channel_paths(EVALUATION_ROOT, 2) + + assert request == f"{EVALUATION_ROOT}/results/attempt-2/judge/request.json" + assert response == f"{EVALUATION_ROOT}/results/attempt-2/judge/response.json" + + +def test_parse_judge_request_normalizes_the_batch_contract() -> None: + request = parse_judge_request(_request_payload()) + + assert request.request_id == "batch-001-002-attempt-1" + assert request.case_ids == ("case-1", "case-2") + assert request.dimensions == ("semantic_fidelity", "output_contract") + assert request.thread_id == "" + assert request.budget_seconds == 0.0 + assert parse_judge_request(_request_payload(budget_seconds=360)).budget_seconds == ( + 360.0 + ) + assert request.case_context[1] == { + "case_id": "case-2", + "state": "failed", + "criteria": False, + "contract": False, + "runtime_observation": False, + } + + +@pytest.mark.parametrize( + "payload", + [ + _request_payload(schema_version=JUDGE_CHANNEL_SCHEMA_VERSION + 1), + _request_payload(request_id=""), + _request_payload(request_id="x" * 129), + _request_payload(prompt=" "), + _request_payload(dimensions=[]), + _request_payload(dimensions=["semantic_fidelity", "semantic_fidelity"]), + _request_payload(dimensions=[1]), + _request_payload(case_context=[]), + _request_payload(case_context=[{"state": "succeeded"}]), + _request_payload(case_context=[{"case_id": ""}]), + _request_payload(case_context=[{"case_id": "case-1"}] * 65), + _request_payload(thread_id="x" * 257), + _request_payload(budget_seconds=0), + _request_payload(budget_seconds=-1), + _request_payload(budget_seconds=True), + _request_payload(budget_seconds="360"), + _request_payload(budget_seconds=24 * 3600 + 1), + "not-an-object", + ], +) +def test_parse_judge_request_rejects_invalid_messages(payload: object) -> None: + with pytest.raises(JudgeRequestError): + parse_judge_request(payload) + + +@pytest.mark.parametrize( + ("payload", "expected"), + [ + # 没有声明窗口：用 Studio 本地上限。 + (_request_payload(), JUDGE_TURN_TIMEOUT_SECONDS), + # 窗口刚好等于上限 + 预留：仍然被上限压住。 + ( + _request_payload( + budget_seconds=JUDGE_TURN_TIMEOUT_SECONDS + JUDGE_TURN_RESERVE_SECONDS + ), + JUDGE_TURN_TIMEOUT_SECONDS, + ), + # 更宽的窗口不会让回合跑得更久。 + (_request_payload(budget_seconds=1000.0), JUDGE_TURN_TIMEOUT_SECONDS), + # 更窄的窗口按比例缩短，先把预留让出来。 + ( + _request_payload(budget_seconds=120.0), + 120.0 - JUDGE_TURN_RESERVE_SECONDS, + ), + ( + _request_payload(budget_seconds=40.0), + 40.0 - JUDGE_TURN_RESERVE_SECONDS, + ), + ], +) +def test_judge_turn_budget_fits_inside_the_declared_window( + payload: dict[str, Any], + expected: float, +) -> None: + assert judge_turn_budget(parse_judge_request(payload)) == pytest.approx(expected) + + +def test_answer_judge_request_defaults_to_the_declared_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[float] = [] + + async def fake_run_judge_turn(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: + calls.append(float(kwargs["timeout_seconds"])) + return _JUDGED_CASES, "thread-new" + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + answer_judge_request( + parse_judge_request(_request_payload(budget_seconds=120)), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + ) + + # 窗口 120s、预留 30s：回合最多只能跑 90s。 + assert calls[0] == pytest.approx(120.0 - JUDGE_TURN_RESERVE_SECONDS, abs=1.0) + + +def test_answer_judge_request_reports_a_slow_turn_as_a_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fake_run_judge_turn(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise ToolTurnDeadlineExceeded("Codex 回合超出时间预算。") + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload(thread_id="thread-old")), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + ) + + assert response["ok"] is False + # 慢回合与「app-server 用不了」必须给出不同的错误码。 + assert response["error"]["code"] == "judge_turn_timeout" # type: ignore[index] + + +def test_answer_judge_request_returns_the_verdict_and_its_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + + async def fake_run_judge_turn(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: + calls.append(kwargs) + return _JUDGED_CASES, "thread-new" + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload(thread_id="thread-old")), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + model="dola-seed", + timeout_seconds=5, + ) + + assert response["ok"] is True + assert response["cases"] == _JUDGED_CASES + assert isinstance(response["cases"], list) + assert response["thread_id"] == "thread-new" + assert response["request_id"] == "batch-001-002-attempt-1" + assert [call["thread_id"] for call in calls] == ["thread-old"] + assert calls[0]["model"] == "dola-seed" + assert calls[0]["cwd"] == "/migration/output/veadk" + assert calls[0]["dimensions"] == ["semantic_fidelity", "output_contract"] + assert calls[0]["case_context"][0]["case_id"] == "case-1" + assert calls[0]["case_context"][1]["state"] == "failed" + assert response["created_at"].endswith("Z") + + +def test_answer_judge_request_retries_a_dead_thread_on_a_fresh_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + + async def fake_run_judge_turn(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: + calls.append(kwargs["thread_id"]) + if kwargs["thread_id"]: + raise JudgeTurnUnavailable("thread is gone") + return [], "thread-fresh" + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload(thread_id="thread-old")), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + timeout_seconds=5, + ) + + assert calls == ["thread-old", ""] + assert response["ok"] is True + assert response["thread_id"] == "thread-fresh" + + +def test_answer_judge_request_starts_one_thread_when_none_is_bound( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + + async def fake_run_judge_turn(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: + calls.append(kwargs["thread_id"]) + return [], "thread-first" + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload()), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + timeout_seconds=5, + ) + + assert calls == [""] + assert response["thread_id"] == "thread-first" + + +def test_answer_judge_request_answers_a_transport_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """回合自己抛出的传输错误也必须变成信封。 + + 否则 runner 收不到任何回应，只能一直等到自己的窗口结束再降级。 + """ + + async def fake_run_judge_turn(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise ToolTurnUnavailable("app-server websocket closed") + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload()), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + ) + + assert response["ok"] is False + assert response["error"]["code"] == "judge_turn_unavailable" # type: ignore[index] + + +def test_answer_judge_request_answers_an_unexpected_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fake_run_judge_turn(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise ValueError("client bug") + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload()), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + ) + + assert response["ok"] is False + assert response["error"]["code"] == "judge_turn_unavailable" # type: ignore[index] + assert "client bug" in response["error"]["message"] # type: ignore[index] + + +def test_answer_judge_request_returns_an_error_envelope_when_no_turn_delivers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fake_run_judge_turn(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise JudgeTurnUnavailable("app-server refused the turn") + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload(thread_id="thread-old")), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + timeout_seconds=5, + ) + + assert response["ok"] is False + assert "cases" not in response + assert response["error"]["code"] == "judge_turn_unavailable" # type: ignore[index] + assert "refused" in response["error"]["message"] # type: ignore[index] + assert response["thread_id"] == "" + + +def test_answer_judge_request_returns_an_error_envelope_without_a_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fake_run_judge_turn(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise AssertionError("an exhausted budget must not start a turn") + + monkeypatch.setattr(judge_driver_module, "run_judge_turn", fake_run_judge_turn) + + response = answer_judge_request( + parse_judge_request(_request_payload()), + endpoint="https://sandbox.invalid", + cwd="/migration/output/veadk", + timeout_seconds=0, + ) + + assert response["ok"] is False + # 预算已经用尽：这不是协议故障，而是这个回合没能在窗口内交付。 + assert response["error"]["code"] == "judge_turn_timeout" # type: ignore[index] + + +def _driver( + gateway: FakeGateway, + monkeypatch: pytest.MonkeyPatch, + handler: Any, + *, + enabled: bool = True, +) -> SandboxJudgeDriver: + monkeypatch.setattr(judge_driver_module, "run_judge_turn", handler) + return SandboxJudgeDriver( + gateway, # type: ignore[arg-type] + cwd="/migration/output/veadk", + enabled=lambda: enabled, + ) + + +def _write_request(gateway: FakeGateway, attempt: int = 1) -> str: + request_path, _ = judge_channel_paths(EVALUATION_ROOT, attempt) + gateway.files[request_path] = json.dumps(_request_payload()).encode("utf-8") + return request_path + + +def test_drive_answers_one_request_once(monkeypatch: pytest.MonkeyPatch) -> None: + gateway = FakeGateway() + calls: list[Any] = [] + + async def handler(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: + calls.append(kwargs["thread_id"]) + return _JUDGED_CASES, "thread-1" + + driver = _driver(gateway, monkeypatch, handler) + _write_request(gateway) + _, response_path = judge_channel_paths(EVALUATION_ROOT, 1) + + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + assert _wait_for(lambda: response_path in gateway.files) + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + time.sleep(0.05) + + assert calls == [""] + response = json.loads(gateway.files[response_path]) + assert response["ok"] is True + assert response["cases"] == _JUDGED_CASES + assert gateway.puts == [response_path] + + +def test_drive_does_nothing_without_a_request(monkeypatch: pytest.MonkeyPatch) -> None: + gateway = FakeGateway() + + async def handler(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise AssertionError("no turn without a request") + + driver = _driver(gateway, monkeypatch, handler) + + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + + assert gateway.files == {} + assert gateway.puts == [] + + +def test_drive_ignores_a_malformed_request(monkeypatch: pytest.MonkeyPatch) -> None: + gateway = FakeGateway() + + async def handler(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise AssertionError("no turn for a malformed request") + + driver = _driver(gateway, monkeypatch, handler) + request_path, _ = judge_channel_paths(EVALUATION_ROOT, 1) + gateway.files[request_path] = b'{"schema_version": 99}' + + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + + assert gateway.puts == [] + + +def test_drive_respects_the_switch(monkeypatch: pytest.MonkeyPatch) -> None: + gateway = FakeGateway() + + async def handler(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise AssertionError("no turn while the channel is off") + + driver = _driver(gateway, monkeypatch, handler, enabled=False) + _write_request(gateway) + + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + + assert gateway.puts == [] + + +def test_drive_keeps_one_turn_in_flight_per_attempt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = FakeGateway() + started = threading.Event() + release = threading.Event() + calls: list[str] = [] + + async def handler(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: + calls.append(kwargs["thread_id"]) + started.set() + release.wait(5) + return _JUDGED_CASES, "thread-1" + + driver = _driver(gateway, monkeypatch, handler) + _write_request(gateway) + + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + assert started.wait(5) + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + release.set() + _, response_path = judge_channel_paths(EVALUATION_ROOT, 1) + + assert _wait_for(lambda: response_path in gateway.files) + assert calls == [""] + + +def test_drive_replays_a_request_after_a_restart( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = FakeGateway() + calls: list[str] = [] + + async def handler(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: + calls.append(kwargs["thread_id"]) + return _JUDGED_CASES, "thread-2" + + _write_request(gateway) + first = _driver(gateway, monkeypatch, handler) + first.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + _, response_path = judge_channel_paths(EVALUATION_ROOT, 1) + assert _wait_for(lambda: response_path in gateway.files) + gateway.files.pop(response_path) + + # 新的 Studio 进程没有任何内存状态：同一个请求会重放，而不是被丢弃。 + second = _driver(gateway, monkeypatch, handler) + second.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + + assert _wait_for(lambda: response_path in gateway.files) + assert calls == ["", ""] + + +def test_drive_reports_a_failed_turn_as_an_error_envelope( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = FakeGateway() + + async def handler(**_kwargs: Any) -> tuple[list[dict[str, Any]], str]: + raise JudgeTurnUnavailable("app-server is down") + + driver = _driver(gateway, monkeypatch, handler) + _write_request(gateway) + + driver.drive(_session(), evaluation_root=EVALUATION_ROOT, attempt=1) + _, response_path = judge_channel_paths(EVALUATION_ROOT, 1) + + assert _wait_for(lambda: response_path in gateway.files) + response = json.loads(gateway.files[response_path]) + assert response["ok"] is False + assert response["error"]["code"] == "judge_turn_unavailable" diff --git a/tests/frontend/server/migration_evaluation/test_runner.py b/tests/frontend/server/migration_evaluation/test_runner.py index 50a61ece4..4370d26d1 100644 --- a/tests/frontend/server/migration_evaluation/test_runner.py +++ b/tests/frontend/server/migration_evaluation/test_runner.py @@ -22,6 +22,16 @@ import pytest +from frontend.server.migration.evaluation.judge_app_server import ( + JudgeContractError, + validate_judge_cases, +) +from frontend.server.migration.evaluation.judge_channel import ( + JUDGE_APP_SERVER_ENV, + JUDGE_CHANNEL_SCHEMA_VERSION, + JUDGE_TOOL_NAME, + judge_channel_paths, +) from frontend.server.migration.evaluation.runner import ( AGENTKIT_CONFIG_MAX_BYTES, AgentkitConfigError, @@ -140,7 +150,10 @@ def test_uploaded_runner_source_compiles_and_has_bounded_security_contracts() -> assert "cloud_credential_path" in source -def test_start_uploads_non_secret_assets_and_background_command() -> None: +def test_start_uploads_non_secret_assets_and_background_command( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv(JUDGE_APP_SERVER_ENV, raising=False) gateway = FakeGateway() role_calls: list[dict[str, object]] = [] @@ -175,6 +188,15 @@ def resolve_runtime_role(**kwargs: object) -> str: assert config["execution_results_path"].endswith( "/attempt-1/execution-results.jsonl" ) + judge_request_path, judge_response_path = judge_channel_paths( + EVALUATION_ROOT, + 1, + ) + assert config["judge_channel"] == { + "request_path": judge_request_path, + "response_path": judge_response_path, + "tool_name": JUDGE_TOOL_NAME, + } assert config["dimension_definitions"][0]["default_weight"] == 1 assert config["remote_write_not_after"] == 1_788_777_600.0 assert config["agentkit_config_protocol"] == "legacy" @@ -200,6 +222,33 @@ def resolve_runtime_role(**kwargs: object) -> str: assert all("cloud-sk" not in command for _, command, _ in gateway.commands) +def test_start_pins_the_scripted_judge_when_the_channel_is_switched_off( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(JUDGE_APP_SERVER_ENV, "0") + gateway = FakeGateway() + runner = SandboxMigrationEvaluationRunner( # type: ignore[arg-type] + gateway, + resolve_credentials=lambda: ("cloud-ak", "cloud-sk", "cloud-token"), + provider="byteplus", + resolve_runtime_role=lambda **_kwargs: SHARED_RUNTIME_ROLE, + ) + + runner.start( + _session(), + task_id=TASK_ID, + attempt=1, + runtime_name="migration-eval-111111111111-a1", + dimensions=["semantic_fidelity"], + dataset_sha256=DATASET_SHA256, + artifact_sha256=ARTIFACT_SHA256, + secret_path=None, + ) + + config = json.loads(gateway.files[f"{EVALUATION_ROOT}/control/runner-1.json"]) + assert config["judge_channel"] is None + + def test_start_localizes_english_judge_configuration() -> None: gateway = FakeGateway() runner = SandboxMigrationEvaluationRunner( # type: ignore[arg-type] @@ -549,6 +598,542 @@ def _observation(text: str) -> dict[str, object]: } +def _channel_config(tmp_path: Path, config: dict[str, Any]) -> dict[str, Any]: + """Point one runner config at a judge channel on disk.""" + channel = tmp_path / "judge" + channel.mkdir(exist_ok=True) + request = channel / "request.json" + response = channel / "response.json" + config["judge_channel"] = { + "request_path": str(request), + "response_path": str(response), + "tool_name": JUDGE_TOOL_NAME, + } + return config + + +def _judged_case(case_id: str) -> dict[str, Any]: + return { + "case_id": case_id, + "dimensions": [ + { + "id": "semantic_fidelity", + "score": 0.8, + "reason": "输出与期望一致", + "evidence": ["输出证据"], + "evidence_sources": ["observed_output"], + "severity": "low", + } + ], + } + + +def _judge_response( + request_id: str, + cases: list[dict[str, Any]], + *, + thread_id: str = "thread-channel", +) -> bytes: + return json.dumps( + { + "schema_version": JUDGE_CHANNEL_SCHEMA_VERSION, + "request_id": request_id, + "ok": True, + "thread_id": thread_id, + "cases": cases, + "created_at": "2026-09-07T10:00:00Z", + } + ).encode("utf-8") + + +def _judge_error_response( + request_id: str, code: str = "judge_turn_unavailable" +) -> bytes: + return json.dumps( + { + "schema_version": JUDGE_CHANNEL_SCHEMA_VERSION, + "request_id": request_id, + "ok": False, + "thread_id": "", + "error": {"code": code, "message": "裁判回合不可用"}, + "created_at": "2026-09-07T10:00:00Z", + } + ).encode("utf-8") + + +def _diagnostics(config: dict[str, Any]) -> list[str]: + path = Path(config["diagnostic_path"]) + if not path.is_file(): + return [] + return [json.loads(line)["event"] for line in path.read_text().splitlines()] + + +def test_runner_source_keeps_the_channel_contract_in_step() -> None: + namespace = _runner_namespace() + + assert namespace["JUDGE_CHANNEL_SCHEMA_VERSION"] == JUDGE_CHANNEL_SCHEMA_VERSION + + +def test_judge_channel_delivers_the_batch_without_running_codex( + tmp_path: Path, +) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + commands: list[list[str]] = [] + + def run_capped(args: list[str], **_kwargs: object) -> tuple[int, bytes, int]: + commands.append(args) + raise AssertionError("the channel must not fall back to codex exec") + + namespace["run_capped"] = run_capped + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_response("batch-001-001-attempt-1", [_judged_case("case-1")]) + ) + + judged = namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert [item["case_id"] for item in judged] == ["case-1"] + assert commands == [] + request = json.loads( + Path(config["judge_channel"]["request_path"]).read_text(encoding="utf-8") + ) + assert request["request_id"] == "batch-001-001-attempt-1" + assert request["batch_start"] == 0 + assert request["batch_end"] == 1 + assert request["case_ids"] == ["case-1"] + assert request["dimensions"] == ["semantic_fidelity"] + assert request["prompt_version"] == namespace["JUDGE_PROMPT_VERSION"] + assert request["budget_seconds"] == namespace["JUDGE_CHANNEL_TIMEOUT"] + assert request["thread_id"] == "" + assert request["case_context"] == [ + { + "case_id": "case-1", + "state": "succeeded", + "criteria": False, + "contract": False, + "runtime_observation": True, + } + ] + assert JUDGE_TOOL_NAME in request["prompt"] + assert "评测用例与观察结果" in request["prompt"] + thread_record = json.loads(Path(config["thread_path"]).read_text()) + assert thread_record["thread_id"] == "thread-channel" + batch_record = json.loads( + (Path(config["batch_root_path"]) / "batch-001-001.json").read_text() + ) + assert batch_record["case_ids"] == ["case-1"] + assert _diagnostics(config) == [] + + +def test_judge_channel_resumes_and_rebinds_the_bound_thread(tmp_path: Path) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + config["dimensions"] = ["semantic_fidelity"] + namespace["run_capped"] = lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("no codex exec") + ) + response_path = Path(config["judge_channel"]["response_path"]) + response_path.write_bytes( + _judge_response( + "batch-001-001-attempt-1", + [_judged_case("case-1")], + thread_id="thread-first", + ) + ) + observations = { + "case-1": _observation("one"), + "case-2": _observation("two"), + } + + namespace["judge_batch"](config, 0, [_case("case-1")], observations, None, {}) + response_path.write_bytes( + _judge_response( + "batch-002-002-attempt-1", + [_judged_case("case-2")], + thread_id="thread-second", + ) + ) + namespace["judge_batch"](config, 1, [_case("case-2")], observations, None, {}) + + request = json.loads( + Path(config["judge_channel"]["request_path"]).read_text(encoding="utf-8") + ) + assert request["thread_id"] == "thread-first" + thread_record = json.loads(Path(config["thread_path"]).read_text()) + assert thread_record["thread_id"] == "thread-second" + + +def test_judge_channel_falls_back_to_codex_exec_on_an_error_envelope( + tmp_path: Path, +) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + commands: list[list[str]] = [] + + def run_capped(args: list[str], **_kwargs: object) -> tuple[int, bytes, int]: + commands.append(args) + events = _judge_events("thread-123", ["case-1"]) + return 0, events, len(events) + + namespace["run_capped"] = run_capped + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_error_response("batch-001-001-attempt-1") + ) + + judged = namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert judged[0]["case_id"] == "case-1" + assert [command[0] for command in commands] == ["codex"] + assert _diagnostics(config) == ["judge_channel_unavailable"] + assert json.loads(Path(config["thread_path"]).read_text())["thread_id"] == ( + "thread-123" + ) + + +def test_judge_channel_rejects_a_verdict_that_is_not_a_case_list( + tmp_path: Path, +) -> None: + """Studio must send the case list itself, not the whole tool-call arguments.""" + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + namespace["run_capped"] = lambda *_args, **_kwargs: ( + 0, + _judge_events("thread-123", ["case-1"]), + 0, + ) + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_response( + "batch-001-001-attempt-1", + {"cases": [_judged_case("case-1")]}, # type: ignore[arg-type] + ) + ) + + judged = namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert judged[0]["case_id"] == "case-1" + assert "judge_channel_output_rejected" in _diagnostics(config) + + +def test_runner_and_studio_agree_on_the_judge_payload_contract(tmp_path: Path) -> None: + """The channel's in-turn validator and the runner's authority must not drift. + + Studio rejects a bad payload inside the turn so Codex can correct itself, and the + runner repeats the checks before caching the batch. A payload both sides disagree + about would either waste a whole batch or cache a verdict the runner refuses. + """ + namespace = _runner_namespace() + config = _judge_config(tmp_path) + config["dimensions"] = ["semantic_fidelity"] + cases = [_case("case-1")] + observations = {"case-1": _observation("hello")} + case_context = namespace["judge_case_context"](cases, observations, None) + valid = _judged_case("case-1") + + def mutate(**changes: Any) -> dict[str, Any]: + payload = json.loads(json.dumps(valid)) + payload["dimensions"][0].update(changes) + return payload + + def studio_accepts(payload: dict[str, Any]) -> bool: + try: + validate_judge_cases( + {"cases": [payload]}, + case_context=case_context, + dimensions=config["dimensions"], + ) + except JudgeContractError: + return False + return True + + def runner_accepts(payload: dict[str, Any]) -> bool: + try: + namespace["validate_judged_cases"]( + config, + cases, + [payload], + observations, + None, + ) + except RuntimeError: + return False + return True + + payloads = { + "valid": valid, + "score above one": mutate(score=1.5), + "score without severity": mutate(score=None), + "severity without score": mutate(severity="unknown"), + "unknown severity": mutate(severity="severe"), + "unknown evidence source": mutate(evidence_sources=["trust_me"]), + "duplicate evidence source": mutate( + evidence_sources=["observed_output", "observed_output"] + ), + "empty reason": mutate(reason=" "), + "evidence not a list": mutate(evidence="输出证据"), + "missing score": mutate(score=None, severity="unknown"), + } + + for label, payload in payloads.items(): + assert studio_accepts(payload) == runner_accepts(payload), label + + assert studio_accepts(valid) is True + assert runner_accepts(valid) is True + assert studio_accepts(mutate(score=1.5)) is False + assert runner_accepts(mutate(evidence_sources=["trust_me"])) is False + + +def test_runner_answers_an_unscored_workflow_dimension_with_na(tmp_path: Path) -> None: + """The runner rewrites an unsupported workflow verdict; Studio rejects it instead. + + The rewrite only ever applies to the scripted judge, because Studio refuses such a + payload inside the turn and asks Codex to answer N/A itself. + """ + namespace = _runner_namespace() + config = _judge_config(tmp_path) + config["dimensions"] = ["workflow_tool_fidelity"] + config["dimension_definitions"][0]["id"] = "workflow_tool_fidelity" + cases = [_case("case-1")] + observations = {"case-1": {**_observation("hello"), "runtime_observation": None}} + payload = _judged_case("case-1") + payload["dimensions"][0]["id"] = "workflow_tool_fidelity" + + returned = namespace["validate_judged_cases"]( + config, + cases, + [payload], + observations, + None, + ) + + dimension = returned[0]["dimensions"][0] + assert dimension["score"] is None + assert dimension["severity"] == "unknown" + assert dimension["evidence"] == [] + + +def test_judge_channel_stays_off_for_the_rest_of_the_run(tmp_path: Path) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + namespace["JUDGE_CHANNEL_TIMEOUT"] = 0.05 + namespace["JUDGE_CHANNEL_POLL_SECONDS"] = 0.01 + commands: list[list[str]] = [] + + def run_capped(args: list[str], **_kwargs: object) -> tuple[int, bytes, int]: + commands.append(args) + case_id = "case-1" if len(commands) == 1 else "case-2" + events = _judge_events("thread-123", [case_id]) + return 0, events, len(events) + + namespace["run_capped"] = run_capped + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_error_response("batch-001-001-attempt-1") + ) + observations = { + "case-1": _observation("one"), + "case-2": _observation("two"), + } + + namespace["judge_batch"](config, 0, [_case("case-1")], observations, None, {}) + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_response("batch-002-002-attempt-1", [_judged_case("case-2")]) + ) + namespace["judge_batch"](config, 1, [_case("case-2")], observations, None, {}) + + # 首批失败后，第二批不再尝试通道：既没有新的通道诊断，也没有第二次等待。 + assert [command[0] for command in commands] == ["codex", "codex"] + assert _diagnostics(config) == ["judge_channel_unavailable"] + + +def test_judge_channel_times_out_and_falls_back_to_codex_exec(tmp_path: Path) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + namespace["JUDGE_CHANNEL_TIMEOUT"] = 0.05 + namespace["JUDGE_CHANNEL_POLL_SECONDS"] = 0.01 + commands: list[list[str]] = [] + + def run_capped(args: list[str], **_kwargs: object) -> tuple[int, bytes, int]: + commands.append(args) + events = _judge_events("thread-123", ["case-1"]) + return 0, events, len(events) + + namespace["run_capped"] = run_capped + + judged = namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert judged[0]["case_id"] == "case-1" + assert [command[0] for command in commands] == ["codex"] + assert _diagnostics(config) == ["judge_channel_timeout"] + + +def test_judge_channel_declares_the_smaller_of_the_two_windows(tmp_path: Path) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + namespace["JUDGE_CHANNEL_TIMEOUT"] = 240 + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_response("batch-001-001-attempt-1", [_judged_case("case-1")]) + ) + + namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + request = json.loads( + Path(config["judge_channel"]["request_path"]).read_text(encoding="utf-8") + ) + assert request["budget_seconds"] == 240 + + +def test_judge_fallback_uses_the_budget_left_after_the_channel(tmp_path: Path) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + namespace["JUDGE_CHANNEL_TIMEOUT"] = 0.15 + namespace["JUDGE_CHANNEL_POLL_SECONDS"] = 0.01 + namespace["JUDGE_TIMEOUT"] = 0.2 + timeouts: list[object] = [] + + def run_capped(_args: list[str], **kwargs: object) -> tuple[int, bytes, int]: + timeouts.append(kwargs["timeout"]) + events = _judge_events("thread-123", ["case-1"]) + return 0, events, len(events) + + namespace["run_capped"] = run_capped + + judged = namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert judged[0]["case_id"] == "case-1" + request = json.loads( + Path(config["judge_channel"]["request_path"]).read_text(encoding="utf-8") + ) + # 声明的是两者中更小的那个：本批还剩 0.2s，通道上限 0.15s。 + assert request["budget_seconds"] == namespace["JUDGE_CHANNEL_TIMEOUT"] + assert _diagnostics(config) == ["judge_channel_timeout"] + # 兜底拿到的是通道用完之后的时间，而不是进入本批时的 0.2s。 + assert len(timeouts) == 1 + assert 0 < float(timeouts[0]) < 0.1 # type: ignore[arg-type] + + +def test_judge_channel_ignores_another_batch_response(tmp_path: Path) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + namespace["JUDGE_CHANNEL_TIMEOUT"] = 0.05 + namespace["JUDGE_CHANNEL_POLL_SECONDS"] = 0.01 + namespace["run_capped"] = lambda *_args, **_kwargs: ( + 0, + _judge_events("thread-123", ["case-1"]), + 0, + ) + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_response("batch-009-009-attempt-1", [_judged_case("case-9")]) + ) + + judged = namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert judged[0]["case_id"] == "case-1" + assert _diagnostics(config) == ["judge_channel_timeout"] + + +def test_judge_channel_rejects_a_verdict_that_violates_the_batch_contract( + tmp_path: Path, +) -> None: + namespace = _runner_namespace() + config = _channel_config(tmp_path, _judge_config(tmp_path)) + namespace["run_capped"] = lambda *_args, **_kwargs: ( + 0, + _judge_events("thread-123", ["case-1"]), + 0, + ) + # 返回了别的用例：Studio 侧应当已经拒绝，这里必须再拒绝一次并回退。 + Path(config["judge_channel"]["response_path"]).write_bytes( + _judge_response("batch-001-001-attempt-1", [_judged_case("case-2")]) + ) + + judged = namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert judged[0]["case_id"] == "case-1" + assert "judge_channel_output_rejected" in _diagnostics(config) + + +def test_judge_without_a_channel_still_uses_codex_exec(tmp_path: Path) -> None: + namespace = _runner_namespace() + config = _judge_config(tmp_path) + config["judge_channel"] = None + commands: list[list[str]] = [] + + def run_capped(args: list[str], **kwargs: object) -> tuple[int, bytes, int]: + commands.append(args) + events = _judge_events("thread-123", ["case-1"]) + assert JUDGE_TOOL_NAME not in str(kwargs["input_text"]) + return 0, events, len(events) + + namespace["run_capped"] = run_capped + + namespace["judge_batch"]( + config, + 0, + [_case("case-1")], + {"case-1": _observation("hello")}, + None, + {}, + ) + + assert [command[0] for command in commands] == ["codex"] + assert not Path(config["judge_channel"] or tmp_path / "judge").exists() + + def test_credential_file_requires_mode_600_and_is_one_shot(tmp_path: Path) -> None: namespace = _runner_namespace() secret = tmp_path / "secret.json" @@ -1125,7 +1710,7 @@ def run_capped(args: list[str], **_kwargs: object) -> tuple[int, bytes, int]: batch_record = json.loads( (Path(config["batch_root_path"]) / "batch-001-001.json").read_text() ) - assert batch_record["prompt_version"] == 3 + assert batch_record["prompt_version"] == 4 assert batch_record["batch_start"] == 0 assert batch_record["batch_end"] == 1 @@ -1250,7 +1835,10 @@ def run_capped(_args: list[str], **kwargs: object) -> tuple[int, bytes, int]: {}, ) - assert timeouts == pytest.approx([300.0, 50.0]) + # 两次重试共享同一个单批预算：第二次只剩第一次用剩的时间。 + assert timeouts == pytest.approx( + [namespace["JUDGE_TIMEOUT"], namespace["JUDGE_TIMEOUT"] - 250.0] + ) def test_judge_resumes_persisted_thread_after_runner_restart(tmp_path: Path) -> None: diff --git a/tests/frontend/server/migration_evaluation/test_service.py b/tests/frontend/server/migration_evaluation/test_service.py index adfc70cd1..e62bf99a0 100644 --- a/tests/frontend/server/migration_evaluation/test_service.py +++ b/tests/frontend/server/migration_evaluation/test_service.py @@ -28,6 +28,7 @@ from frontend.server.migration.evaluation.repository import EvaluationAssetMetadata from frontend.server.migration.evaluation.service import ( EVALUATION_DATASET_MANIFEST_PATH, + EVALUATION_ROOT, EVALUATION_REPORT_PATH, EVALUATION_RUNNER_DIAGNOSTICS_ROOT, EVALUATION_SECRET_PATH, @@ -233,7 +234,29 @@ def stop( self.stops.append(attempt) -def _service(*, remaining: int = 3600): +class FakeJudgeDriver: + def __init__(self, *, error: Exception | None = None) -> None: + self.calls: list[tuple[str, int]] = [] + self.error = error + + def drive( + self, + session: MigrationSandboxSession, + *, + evaluation_root: str, + attempt: int, + ) -> None: + assert session.task_id == TASK_ID + self.calls.append((evaluation_root, attempt)) + if self.error is not None: + raise self.error + + +def _service( + *, + remaining: int = 3600, + judge_driver: FakeJudgeDriver | None = None, +): migration = FakeMigration() gateway = FakeGateway(remaining=remaining) repository = FakeRepository() @@ -243,6 +266,7 @@ def _service(*, remaining: int = 3600): gateway, # type: ignore[arg-type] repository=repository, runner=runner, + judge_driver=judge_driver, # type: ignore[arg-type] clock=lambda: NOW, ) return service, migration, gateway, repository, runner @@ -467,6 +491,43 @@ def test_new_remote_writes_are_blocked_below_twenty_minutes() -> None: assert runner.starts == [] +def test_active_evaluation_drives_the_judge_channel_every_tick() -> None: + driver = FakeJudgeDriver() + service, migration, _gateway, _repository, _runner = _service(judge_driver=driver) + service.put_dataset(TASK_ID, "owner", _body()) + _ready(migration) + + service.advance(TASK_ID, "owner") + assert driver.calls == [] + + service.advance(TASK_ID, "owner") + service.advance(TASK_ID, "owner") + + assert driver.calls == [(EVALUATION_ROOT, 1), (EVALUATION_ROOT, 1)] + + +def test_judge_channel_failure_never_stops_the_watcher_tick() -> None: + driver = FakeJudgeDriver(error=RuntimeError("app-server is down")) + service, migration, gateway, _repository, _runner = _service(judge_driver=driver) + service.put_dataset(TASK_ID, "owner", _body()) + _ready(migration) + service.advance(TASK_ID, "owner") + + service.advance(TASK_ID, "owner") + gateway.files[f"{EVALUATION_RUNNER_DIAGNOSTICS_ROOT}/runner-1-exit.json"] = ( + json.dumps( + {"schema_version": 1, "exit_code": 17, "finished_at": int(NOW)} + ).encode() + ) + service.advance(TASK_ID, "owner") + + # 每一次活跃 tick 都会尝试接管，包括随后判定 runner 退出的那一次。 + assert driver.calls == [(EVALUATION_ROOT, 1), (EVALUATION_ROOT, 1)] + snapshot = service.snapshot(TASK_ID, "owner") + assert snapshot["state"] == "failed" + assert snapshot["error"]["code"] == "MIGRATION_EVALUATION_RUNNER_EXITED" # type: ignore[index] + + def test_finished_runner_cannot_leave_an_active_evaluation_stuck() -> None: service, migration, gateway, _repository, runner = _service() service.put_dataset(TASK_ID, "owner", _body()) diff --git a/tests/frontend/test_migration_codex_tool_turn.py b/tests/frontend/test_migration_codex_tool_turn.py new file mode 100644 index 000000000..ab3f616c9 --- /dev/null +++ b/tests/frontend/test_migration_codex_tool_turn.py @@ -0,0 +1,228 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the shared one-turn dynamic-tool helper.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from frontend.server.migration import codex_tool_turn +from frontend.server.migration.codex_tool_turn import ( + ToolTurnDeadlineExceeded, + ToolTurnUnavailable, + run_tool_turn, +) +from veadk.cli.codex_app_server import CodexAppServerError + + +class _FakeSession: + """A Codex app-server session that streams scripted events.""" + + def __init__( + self, + events: list[str], + *, + connect_error: Exception | None = None, + stream_error: Exception | None = None, + forever: bool = False, + thread_id: str = "thread-1", + ) -> None: + self.thread_id = thread_id + self.cwd = "" + self.model = "" + self.interrupted = 0 + self.closed = 0 + self.attached: list[str] = [] + self.tools: list[str] = [] + self._events = events + self._connect_error = connect_error + self._stream_error = stream_error + self._forever = forever + + def register_dynamic_tool(self, name, description, schema, handler) -> None: + assert description + assert schema == {} + assert callable(handler) + self.tools.append(name) + + async def connect(self) -> None: + if self._connect_error is not None: + raise self._connect_error + + async def attach_thread(self, thread_id: str) -> None: + if self._connect_error is not None: + raise self._connect_error + self.attached.append(thread_id) + self.thread_id = thread_id + + async def stream_turn( + self, + _prompt: str, + *, + timeout_seconds: float, + ) -> AsyncIterator[str]: + assert timeout_seconds > 0 + delivered = list(self._events) + while True: + for event in delivered: + await asyncio.sleep(0.01) + yield event + if not self._forever: + break + delivered = ["keep-going"] + if self._stream_error is not None: + # 先交付事件，再让连接失败：结果已经到手时不能丢掉它。 + raise self._stream_error + + async def interrupt(self) -> None: + self.interrupted += 1 + + async def close(self) -> None: + self.closed += 1 + + +def _install( + monkeypatch: pytest.MonkeyPatch, + session: _FakeSession, +) -> _FakeSession: + monkeypatch.setattr( + codex_tool_turn, + "CodexAppServerSession", + lambda _endpoint: session, + ) + return session + + +async def _run(session: _FakeSession, **overrides: Any) -> str: + kwargs: dict[str, Any] = { + "endpoint": "https://sandbox.invalid", + "prompt": "judge this batch", + "cwd": "/migration/output/veadk", + "tool_name": "reportEvaluation", + "tool_description": "提交判定结果。", + "tool_schema": {}, + "handler": lambda _arguments: {"success": True}, + "has_result": lambda: False, + "timeout_seconds": 5.0, + } + kwargs.update(overrides) + return await run_tool_turn(**kwargs) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_run_tool_turn_stops_as_soon_as_the_result_arrives( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _install( + monkeypatch, + _FakeSession(["one", "two", "three"], thread_id="thread-live"), + ) + seen: list[str] = [] + + async def run() -> str: + return await _run( + session, + event_sink=lambda event: seen.append(str(event)), + has_result=lambda: len(seen) == 2, + ) + + assert await run() == "thread-live" + # 结果到手就打断回合，后面的进展事件不再消费。 + assert seen == ["one", "two"] + assert session.interrupted == 1 + assert session.closed == 1 + assert session.tools == ["reportEvaluation"] + + +@pytest.mark.asyncio +async def test_run_tool_turn_resumes_the_bound_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _install(monkeypatch, _FakeSession(["one"])) + + returned = await _run(session, thread_id="thread-bound", has_result=lambda: True) + + assert session.attached == ["thread-bound"] + assert returned == "thread-bound" + + +@pytest.mark.asyncio +async def test_run_tool_turn_reports_a_transport_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _install( + monkeypatch, + _FakeSession([], connect_error=CodexAppServerError("app-server 未启动")), + ) + + with pytest.raises(ToolTurnUnavailable) as error: + await _run(session) + + assert not isinstance(error.value, ToolTurnDeadlineExceeded) + assert "app-server 未启动" in str(error.value) + assert session.closed == 1 + + +@pytest.mark.asyncio +async def test_run_tool_turn_reports_the_client_timeout_as_a_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _install(monkeypatch, _FakeSession([], stream_error=TimeoutError())) + + with pytest.raises(ToolTurnDeadlineExceeded): + await _run(session) + + +@pytest.mark.asyncio +async def test_run_tool_turn_stops_at_the_wall_clock_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 事件一直在来（空闲超时永远不会触发），但已经超出调用方的窗口。 + session = _install(monkeypatch, _FakeSession(["progress"], forever=True)) + + with pytest.raises(ToolTurnDeadlineExceeded): + await _run(session, timeout_seconds=0.05) + + assert session.interrupted == 1 + assert session.closed == 1 + + +@pytest.mark.asyncio +async def test_run_tool_turn_keeps_a_result_delivered_before_the_stream_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _install( + monkeypatch, + _FakeSession( + ["one"], + stream_error=CodexAppServerError("连接中断"), + thread_id="thread-delivered", + ), + ) + seen: list[str] = [] + + async def run() -> str: + return await _run( + session, + event_sink=lambda event: seen.append(str(event)), + has_result=lambda: bool(seen), + ) + + assert await run() == "thread-delivered" + assert session.interrupted == 1 diff --git a/veadk/cli/cli_frontend.py b/veadk/cli/cli_frontend.py index 1c04f2d9a..a1512c473 100644 --- a/veadk/cli/cli_frontend.py +++ b/veadk/cli/cli_frontend.py @@ -3483,10 +3483,14 @@ def _sandbox_is_admin(request: Request) -> bool: return _request_role(request).is_admin from frontend.server.migration.gateway import MigrationSandboxGateway + from frontend.server.migration.evaluation.judge_driver import ( + SandboxJudgeDriver, + ) from frontend.server.migration.evaluation.repository import ( TosMigrationEvaluationRepository, ) from frontend.server.migration.evaluation.runner import ( + EVALUATION_PROJECT_PATH, SandboxMigrationEvaluationRunner, ) from frontend.server.migration.evaluation.service import ( @@ -3547,6 +3551,10 @@ def _migration_creator(request: Request) -> str: resolve_credentials=_resolve_ve_credentials, provider=provider, ), + judge_driver=SandboxJudgeDriver( + migration_gateway, + cwd=EVALUATION_PROJECT_PATH, + ), ) if not is_vestack_deployment: from frontend.server.workspace_tool import mount_workspace_upgrade_repair From db32fe404e2717c7b660b2dbbe2313702f821bb9 Mon Sep 17 00:00:00 2001 From: "wujiaming.ai" Date: Mon, 21 Sep 2026 19:46:56 +0800 Subject: [PATCH 03/16] fix(studio): keep migration analysis, questions and delivery on app-server turns The migration workspace polled the Sandbox, re-ran the whole analysis to ask the user a question, and left a killed delivery hanging on a lease nobody watched. This puts the long work on Studio-driven Codex turns and gives the page a stream to follow. - Stream the task: an append-only event log plus SSE replaces polling, so a refresh or a second tab resumes from a cursor and progress no longer needs an open page (`events.py`, `routes.py`, `MigrationWorkspace.tsx`). - Answer in place: the analysis turn registers its own `askUser` dynamic tool and waits for the page, so one attempt finishes the analysis instead of a `needs_input` re-run (`analysis_input.py`, `models.py`, `analysis_input` routes). - Lease the delivery: the in-Sandbox driver publishes a heartbeat lease, and a killed run settles as `MIGRATION_DELIVERY_INTERRUPTED` instead of hanging forever. - Close the delivery on a turn: `publishArtifact` re-reads the delivered zip and requires it to match the CLI manifest, and `reportDelivery` may only explain the state the Sandbox published, never turn a failure into a success (`delivery_turn.py`, `contracts.py`, `service.py`). - Show app-server work: turn events are translated back into the `codex exec --json` shape the page already parses, so the activity feed is no longer empty on the app-server driver (`activity.py`). Verified: `pytest tests/frontend -k migration` (517 passed), `node --test frontend/tests/*.test.mjs`, `npx tsc --noEmit`, `ruff check .`, and local end-to-end runs for the analysis, question, delivery and killed-driver paths. --- frontend/server/migration/activity.py | 289 +++++ frontend/server/migration/analysis_input.py | 384 ++++++ frontend/server/migration/app_server.py | 107 +- frontend/server/migration/codex_tool_turn.py | 81 +- frontend/server/migration/contracts.py | 111 ++ frontend/server/migration/delivery_turn.py | 401 +++++++ frontend/server/migration/events.py | 388 ++++++ frontend/server/migration/models.py | 34 + frontend/server/migration/routes.py | 173 ++- frontend/server/migration/service.py | 1063 ++++++++++++++++- frontend/src/adk/migrations.ts | 264 ++++ frontend/src/i18n/resources/en-US/adk.json | 10 +- .../src/i18n/resources/en-US/migrations.json | 9 + frontend/src/i18n/resources/zh-CN/adk.json | 10 +- .../src/i18n/resources/zh-CN/migrations.json | 9 + .../src/migrations/MigrationWorkspace.css | 107 ++ .../src/migrations/MigrationWorkspace.tsx | 295 +++-- frontend/tests/migrationEvents.test.mjs | 297 +++++ frontend/tests/migrationPendingInput.test.mjs | 186 +++ frontend/tests/migrationWorkspace.test.mjs | 53 +- tests/frontend/test_migration_activity.py | 248 ++++ .../frontend/test_migration_analysis_input.py | 376 ++++++ tests/frontend/test_migration_app_server.py | 117 ++ .../test_migration_codex_tool_turn.py | 52 + tests/frontend/test_migration_contracts.py | 224 ++++ .../frontend/test_migration_delivery_turn.py | 344 ++++++ tests/frontend/test_migration_events.py | 567 +++++++++ tests/frontend/test_migration_routes.py | 32 + tests/frontend/test_migration_server.py | 760 ++++++++++++ .../frontend/test_migration_start_protocol.py | 270 +++++ 30 files changed, 7136 insertions(+), 125 deletions(-) create mode 100644 frontend/server/migration/activity.py create mode 100644 frontend/server/migration/analysis_input.py create mode 100644 frontend/server/migration/delivery_turn.py create mode 100644 frontend/server/migration/events.py create mode 100644 frontend/tests/migrationEvents.test.mjs create mode 100644 frontend/tests/migrationPendingInput.test.mjs create mode 100644 tests/frontend/test_migration_activity.py create mode 100644 tests/frontend/test_migration_analysis_input.py create mode 100644 tests/frontend/test_migration_delivery_turn.py create mode 100644 tests/frontend/test_migration_events.py create mode 100644 tests/frontend/test_migration_start_protocol.py diff --git a/frontend/server/migration/activity.py b/frontend/server/migration/activity.py new file mode 100644 index 000000000..527e6d757 --- /dev/null +++ b/frontend/server/migration/activity.py @@ -0,0 +1,289 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Record an app-server analysis turn as the activity log the page already reads. + +The scripted driver gets this log for free: ``codex exec --json`` writes its event +stream inside the Sandbox, and ``MigrationService.activity`` parses it into the items +the migration page renders. An app-server turn never writes that file, so analysis on +the app-server driver showed an empty activity feed even though Codex was working. + +Rather than teach the page a second source, this module translates the app-server's +typed events back into the same ``codex exec --json`` line shape, so exactly one reader +(``_parse_activity_log``) serves both drivers and the Sandbox stays the source of truth. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +import contextlib +import json +import logging + +from veadk.cli.codex_app_server import CodexAppServerEvent + +logger = logging.getLogger(__name__) + +DEFAULT_FLUSH_SECONDS = 2.0 +DEFAULT_MAX_BYTES = 4 * 1024 * 1024 +_MAX_TEXT_CHARS = 120_000 +_TRUNCATED_MARKER = "\n…内容已截断" + +_COMPLETED_STATUSES = {"completed", "done"} +_FAILED_STATUSES = {"failed", "error", "declined"} + +_TOOL_ITEM_TYPES = { + "commandExecution": "command_execution", + "fileChange": "file_change", + "mcpToolCall": "mcp_tool_call", + "webSearch": "web_search", +} + +_TODO_STATUSES = { + "completed": "completed", + "done": "completed", + "inprogress": "in_progress", + "in_progress": "in_progress", + "running": "in_progress", + "failed": "failed", + "error": "failed", +} + + +def _text(value: object, limit: int = _MAX_TEXT_CHARS) -> str: + if not isinstance(value, str): + return "" + return value[:limit] + + +def _bounded(value: object, limit: int = _MAX_TEXT_CHARS) -> object: + """Keep a payload small; the parser truncates and redacts what it renders.""" + if isinstance(value, str): + return value[:limit] + return value + + +class AnalysisActivityLog: + """Turn an app-server turn's events into an append-only analysis activity log. + + ``record`` is the turn's event sink, so it must stay cheap: events are buffered and + the Sandbox file is replaced as a whole by ``flush``, which the caller runs on a + timer and once more when the turn ends. A flush is best-effort by design — an + activity feed must never fail an analysis. + """ + + def __init__( + self, + write: Callable[[bytes], None], + *, + flush_seconds: float = DEFAULT_FLUSH_SECONDS, + max_bytes: int = DEFAULT_MAX_BYTES, + ) -> None: + self._write = write + self._flush_seconds = max(0.1, flush_seconds) + self._max_bytes = max(1, max_bytes) + self._lines: list[str] = [] + self._dirty = False + self._texts: dict[str, str] = {} + self._outputs: dict[str, str] = {} + + @property + def lines(self) -> list[str]: + return list(self._lines) + + def line(self, event: CodexAppServerEvent) -> dict[str, object] | None: + """One ``codex exec --json`` line for ``event``; ``None`` when it has no item.""" + item = self._item(event) + if item is None: + return None + return {"type": self._event_type(event), "item": item} + + def record(self, event: object) -> None: + if not isinstance(event, CodexAppServerEvent): + return + line = self.line(event) + if line is None: + return + self._lines.append(json.dumps(line, ensure_ascii=False)) + self._dirty = True + + def flush(self) -> None: + """Replace the Sandbox log with everything recorded so far.""" + if not self._dirty: + return + self._write(self._content()) + self._dirty = False + + def close(self) -> None: + try: + self.flush() + except Exception as error: # noqa: BLE001 - the feed is not the analysis result + logger.warning( + "Studio migration activity log could not be written error_type=%s", + type(error).__name__, + ) + + async def run(self) -> None: + """Flush on a timer for as long as the caller keeps this task alive.""" + while True: + await asyncio.sleep(self._flush_seconds) + with contextlib.suppress(Exception): + await asyncio.to_thread(self.flush) + + async def aclose(self) -> None: + await asyncio.to_thread(self.close) + + def _content(self) -> bytes: + lines = list(self._lines) + size = sum(len(line) + 1 for line in lines) + while lines and size > self._max_bytes: + size -= len(lines.pop(0)) + 1 + return ("\n".join(lines) + "\n").encode("utf-8") if lines else b"" + + def _event_type(self, event: CodexAppServerEvent) -> str: + status = str(event.status or "").lower() + if status in _FAILED_STATUSES: + return "item.failed" + if status in _COMPLETED_STATUSES: + return "item.completed" + return "item.updated" + + def _item(self, event: CodexAppServerEvent) -> dict[str, object] | None: + item_id = str(event.item_id or "") + kind = str(event.kind or "") + if kind == "thinking": + return self._message_item(item_id, "reasoning", event.text, append=False) + if kind == "commentary": + return self._message_item( + item_id, "agent_message", event.text, append=False + ) + if kind == "text": + # Live deltas: the reader keeps the last text it saw for an item. + return self._message_item(item_id, "agent_message", event.text, append=True) + if kind in {"text_snapshot", "assistant_final"}: + return self._message_item( + item_id, "agent_message", event.text, append=False + ) + if kind == "tool": + return self._tool_item(item_id, event) + if kind == "tool_output": + return self._output_item(item_id, event) + if kind == "plan": + return self._plan_item(event) + return None + + def _message_item( + self, + item_id: str, + item_type: str, + text: str, + *, + append: bool, + ) -> dict[str, object] | None: + value = _text(text) + if not value: + return None + if append and item_id: + value = self._append(self._texts, item_id, value) + elif item_id: + self._texts[item_id] = value + return {"id": item_id, "type": item_type, "text": value} + + def _output_item( + self, item_id: str, event: CodexAppServerEvent + ) -> dict[str, object] | None: + value = _text(event.text) + if not value: + return None + if item_id: + value = self._append(self._outputs, item_id, value) + return { + "id": item_id, + "type": "command_execution", + "status": str(event.status or "running") or "running", + "aggregated_output": value, + } + + @staticmethod + def _append(store: dict[str, str], key: str, value: str) -> str: + combined = f"{store.get(key, '')}{value}" + if len(combined) > _MAX_TEXT_CHARS: + combined = f"{_TRUNCATED_MARKER}\n{combined[-_MAX_TEXT_CHARS:]}" + store[key] = combined + return combined + + def _tool_item( + self, + item_id: str, + event: CodexAppServerEvent, + ) -> dict[str, object] | None: + item_type = _TOOL_ITEM_TYPES.get(str(event.item_type or "")) + if item_type is None: + # Studio's own dynamic tools are the turn's contract, not page content. + return None + arguments = event.arguments if isinstance(event.arguments, dict) else {} + response = event.response if isinstance(event.response, dict) else {} + status = str(event.status or "running") or "running" + item: dict[str, object] = {"id": item_id, "type": item_type, "status": status} + if item_type == "command_execution": + command = _text(arguments.get("command"), 20_000) + if command: + item["command"] = command + output = response.get("output") + if output not in (None, ""): + item["aggregated_output"] = _bounded(output) + exit_code = response.get("exitCode") + if isinstance(exit_code, int) and not isinstance(exit_code, bool): + item["exit_code"] = exit_code + if output in (None, ""): + item["aggregated_output"] = self._outputs.get(item_id, "") + return item + if item_type == "file_change": + item["changes"] = _bounded(arguments.get("changes")) + return item + if item_type == "mcp_tool_call": + server, _, tool = ( + str(event.name or "").removeprefix("MCP · ").partition("/") + ) + item["server"] = server + item["tool"] = tool + item["arguments"] = _bounded(arguments) + if response: + item["result"] = _bounded(response) + return item + item["query"] = _text(arguments.get("query"), 4_000) + return item + + def _plan_item(self, event: CodexAppServerEvent) -> dict[str, object] | None: + steps = event.response if isinstance(event.response, list) else [] + todos: list[dict[str, object]] = [] + for step in steps: + if not isinstance(step, dict): + continue + raw = str(step.get("status") or "").lower() + text = _text(step.get("step"), 4_000) + if not text: + continue + todos.append({"text": text, "status": _TODO_STATUSES.get(raw, "pending")}) + if not todos: + return None + return { + "id": f"plan-{event.turn_id}" if event.turn_id else "plan", + "type": "todo_list", + "items": todos, + } + + +__all__ = ["AnalysisActivityLog", "DEFAULT_FLUSH_SECONDS", "DEFAULT_MAX_BYTES"] diff --git a/frontend/server/migration/analysis_input.py b/frontend/server/migration/analysis_input.py new file mode 100644 index 000000000..6b17f1ef8 --- /dev/null +++ b/frontend/server/migration/analysis_input.py @@ -0,0 +1,384 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Questions a running analysis turn asks the user through Studio. + +Codex' own ``request_user_input`` tool exists in Plan mode only, so the read-only +analysis cannot use it: switching collaboration mode would rewrite the agent's +instructions and disable tools the analysis relies on. The analysis turn therefore +registers its own ``askUser`` dynamic tool. Its handler publishes the questions, +waits for the browser to answer, and returns the answers as the tool result, so one +attempt finishes the whole analysis instead of re-running it from scratch. + +The question and answer shapes mirror Codex' native protocol on purpose: questions are +``{id, header, question, options[]}`` and answers are ``{"": {"answers": +[...]}}``. Transport and protocol therefore stay interchangeable. +""" + +from __future__ import annotations + +import re +import threading +import time +import uuid +from collections.abc import Callable, Iterable +from concurrent.futures import Future, InvalidStateError +from dataclasses import dataclass + +ASK_TOOL_NAME = "askUser" +ASK_TOOL_DESCRIPTION = ( + "在只读分析过程中向用户提出必须由用户决定的问题，并等待用户回答。" + "一次提出 1-3 个问题，每个问题给出简短 header 和完整 question；" + "有自然选择时给出 2-3 个 options（每个含 label 和 description，第一项为推荐项），" + "没有自然选择时省略 options，用户会直接填写。" + "只能提问项目内容无法回答、且答案会改变迁移方式或迁移范围的问题；" + "能自己从项目里查到的事实必须自己查。" +) + +# The native protocol asks for at most three questions; a longer list is a +# questionnaire, which is a different product decision. +MAX_QUESTIONS = 3 +MAX_OPTIONS = 6 +MAX_HEADER_LENGTH = 24 +MAX_QUESTION_LENGTH = 1_000 +MAX_OPTION_LABEL_LENGTH = 60 +MAX_OPTION_DESCRIPTION_LENGTH = 200 +MAX_ANSWER_LENGTH = 4_000 +MAX_ANSWERS_PER_QUESTION = 8 +# An entry nothing ever settles would keep a card on the page forever. +MAX_PENDING_SECONDS = 3_600.0 + +_QUESTION_ID_RE = re.compile(r"[A-Za-z0-9_.-]{1,64}\Z") + +Answers = dict[str, tuple[str, ...]] + + +class AnalysisAskError(ValueError): + """The ``askUser`` call cannot be shown to the user as written.""" + + +def _text( + value: object, + *, + maximum: int, + field: str, + allow_empty: bool = False, +) -> str: + if not isinstance(value, str): + raise AnalysisAskError(f"{field} 必须是字符串") + text = value.strip() + if not text and not allow_empty: + raise AnalysisAskError(f"{field} 不能为空") + if len(text) > maximum: + raise AnalysisAskError(f"{field} 不能超过 {maximum} 个字符") + return text + + +def _options(value: object) -> list[dict[str, str]]: + if value is None: + return [] + if not isinstance(value, list): + raise AnalysisAskError("options 必须是数组") + if len(value) > MAX_OPTIONS: + raise AnalysisAskError(f"options 不能超过 {MAX_OPTIONS} 项") + options: list[dict[str, str]] = [] + labels: set[str] = set() + for item in value: + if not isinstance(item, dict): + raise AnalysisAskError("options 的每一项都必须包含 label 和 description") + label = _text( + item.get("label"), + maximum=MAX_OPTION_LABEL_LENGTH, + field="options.label", + ) + if label in labels: + raise AnalysisAskError("options 的 label 不能重复") + labels.add(label) + options.append( + { + "label": label, + "description": _text( + item.get("description"), + maximum=MAX_OPTION_DESCRIPTION_LENGTH, + field="options.description", + ), + } + ) + return options + + +def normalize_questions(arguments: object) -> tuple[dict[str, object], ...]: + """Validate one ``askUser`` call and return the questions the page renders.""" + if not isinstance(arguments, dict): + raise AnalysisAskError("参数必须是对象") + raw = arguments.get("questions") + if not isinstance(raw, list) or not raw: + raise AnalysisAskError("questions 必须是非空数组") + if len(raw) > MAX_QUESTIONS: + raise AnalysisAskError(f"questions 不能超过 {MAX_QUESTIONS} 个") + questions: list[dict[str, object]] = [] + seen: set[str] = set() + for item in raw: + if not isinstance(item, dict): + raise AnalysisAskError("questions 的每一项都必须是对象") + question_id = _text(item.get("id"), maximum=64, field="questions.id") + if not _QUESTION_ID_RE.match(question_id): + raise AnalysisAskError( + "questions.id 只能包含字母、数字、下划线、点或连字符，且不超过 64 个字符" + ) + if question_id in seen: + raise AnalysisAskError("questions.id 不能重复") + seen.add(question_id) + questions.append( + { + "id": question_id, + "header": _text( + item.get("header"), + maximum=MAX_HEADER_LENGTH, + field="questions.header", + ), + "question": _text( + item.get("question"), + maximum=MAX_QUESTION_LENGTH, + field="questions.question", + ), + "options": _options(item.get("options")), + } + ) + return tuple(questions) + + +def _answers(value: object) -> Answers: + """Validate the answers the browser submits for one question set.""" + if not isinstance(value, dict) or not value: + raise AnalysisAskError("answers 必须是非空对象") + answers: Answers = {} + for raw_id, raw_answer in value.items(): + question_id = _text(raw_id, maximum=64, field="answers 的键") + values = raw_answer if isinstance(raw_answer, list) else [raw_answer] + if not values or len(values) > MAX_ANSWERS_PER_QUESTION: + raise AnalysisAskError("answers 的每项必须包含 1-8 个回答") + collected: list[str] = [] + for entry in values: + text = _text( + entry, + maximum=MAX_ANSWER_LENGTH, + field="answers 的值", + allow_empty=True, + ) + if text: + collected.append(text) + if not collected: + raise AnalysisAskError("answers 的值不能为空") + answers[question_id] = tuple(collected) + return answers + + +def normalize_answers(value: object) -> Answers: + """Public wrapper so routes and tests validate answers the same way.""" + return _answers(value) + + +ASK_TOOL_SCHEMA: dict[str, object] = { + "type": "object", + "additionalProperties": False, + "required": ["questions"], + "properties": { + "questions": { + "type": "array", + "minItems": 1, + "maxItems": MAX_QUESTIONS, + "items": { + "type": "object", + "additionalProperties": False, + "required": ["id", "header", "question"], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[A-Za-z0-9_.-]+$", + }, + "header": { + "type": "string", + "minLength": 1, + "maxLength": MAX_HEADER_LENGTH, + }, + "question": { + "type": "string", + "minLength": 1, + "maxLength": MAX_QUESTION_LENGTH, + }, + "options": { + "type": "array", + "minItems": 1, + "maxItems": MAX_OPTIONS, + "items": { + "type": "object", + "additionalProperties": False, + "required": ["label", "description"], + "properties": { + "label": { + "type": "string", + "minLength": 1, + "maxLength": MAX_OPTION_LABEL_LENGTH, + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": MAX_OPTION_DESCRIPTION_LENGTH, + }, + }, + }, + }, + }, + }, + } + }, +} + + +@dataclass(frozen=True) +class PendingAnalysisInput: + """One question set that an analysis turn is waiting on.""" + + request_id: str + attempt: int + questions: tuple[dict[str, object], ...] + created_at: float + future: Future[Answers | None] + + +def ask_payload(pending: PendingAnalysisInput) -> dict[str, object]: + """The question set as the page renders it.""" + return { + "id": pending.request_id, + "questions": [dict(question) for question in pending.questions], + } + + +class AnalysisInputRegistry: + """Track the questions each Sandbox session's analysis turn is waiting on. + + The analysis turn runs on its own event loop inside a Studio background worker, + while the answer arrives on an HTTP request thread, so the hand-off is a + ``concurrent.futures.Future``: it is settled from any thread and awaited from the + worker's loop. Settling with ``None`` is the "no answer" outcome, which the turn + turns into the ``needs_input`` fallback instead of a hung tool call. + """ + + def __init__( + self, + *, + clock: Callable[[], float] = time.monotonic, + max_pending_seconds: float = MAX_PENDING_SECONDS, + ) -> None: + self._clock = clock + self._max_pending_seconds = max_pending_seconds + self._lock = threading.Lock() + self._pending: dict[str, PendingAnalysisInput] = {} + + def open( + self, + session_id: str, + *, + attempt: int, + questions: Iterable[dict[str, object]], + ) -> PendingAnalysisInput: + """Publish one question set, releasing whatever was pending before.""" + pending = PendingAnalysisInput( + request_id=uuid.uuid4().hex, + attempt=attempt, + questions=tuple(dict(question) for question in questions), + created_at=self._clock(), + future=Future(), + ) + with self._lock: + previous = self._pending.pop(session_id, None) + self._pending[session_id] = pending + if previous is not None: + _settle(previous.future, None) + return pending + + def pending(self, session_id: str) -> PendingAnalysisInput | None: + """The live question set for one session, or ``None``.""" + with self._lock: + pending = self._pending.get(session_id) + if pending is None: + return None + if pending.future.done() or self._is_stale(pending): + self._pending.pop(session_id, None) + return None + return pending + + def resolve( + self, + session_id: str, + *, + request_id: str, + answers: Answers, + ) -> bool: + """Deliver answers to the waiting turn. ``False`` means the ask is gone.""" + with self._lock: + pending = self._pending.get(session_id) + if ( + pending is None + or pending.request_id != request_id + or not _settle(pending.future, answers) + ): + return False + self._pending.pop(session_id, None) + return True + + def discard(self, session_id: str, *, request_id: str = "") -> None: + """Withdraw one question set and release anyone still waiting on it.""" + with self._lock: + pending = self._pending.get(session_id) + if pending is None: + return + if request_id and pending.request_id != request_id: + return + self._pending.pop(session_id, None) + _settle(pending.future, None) + + def _is_stale(self, pending: PendingAnalysisInput) -> bool: + if self._max_pending_seconds <= 0: + return False + return self._clock() - pending.created_at > self._max_pending_seconds + + +def _settle(future: Future[Answers | None], value: Answers | None) -> bool: + """Settle one pending hand-off, tolerating a waiter that already gave up.""" + if future.done(): + return False + try: + future.set_result(value) + except InvalidStateError: + return False + return True + + +__all__ = [ + "ASK_TOOL_DESCRIPTION", + "ASK_TOOL_NAME", + "ASK_TOOL_SCHEMA", + "MAX_PENDING_SECONDS", + "AnalysisAskError", + "AnalysisInputRegistry", + "Answers", + "PendingAnalysisInput", + "ask_payload", + "normalize_answers", + "normalize_questions", +] diff --git a/frontend/server/migration/app_server.py b/frontend/server/migration/app_server.py index 87c71290c..159ba5481 100644 --- a/frontend/server/migration/app_server.py +++ b/frontend/server/migration/app_server.py @@ -19,26 +19,56 @@ update, a commentary message, or a Markdown-fenced reply can no longer be mistaken for the result. Rejections are returned to Codex as ``success: false`` so the same turn can correct itself instead of failing the whole migration. + +The same turn also carries ``askUser``. When the project cannot answer a question that +changes the migration, Codex asks the user inside the turn and keeps analysing with the +answer, which is cheaper and more accurate than re-running the analysis afterwards. """ from __future__ import annotations -from collections.abc import Callable +import json +import logging import os +from collections.abc import Awaitable, Callable from veadk.cli.codex_app_server import CodexDynamicToolResult -from .codex_tool_turn import ToolTurnUnavailable, run_tool_turn +from .analysis_input import ( + ASK_TOOL_DESCRIPTION, + ASK_TOOL_NAME, + ASK_TOOL_SCHEMA, + AnalysisAskError, + Answers, + normalize_questions, +) +from .codex_tool_turn import DynamicTool, ToolTurnUnavailable, run_tool_turn from .contracts import MigrationContractError, validate_analysis_result +logger = logging.getLogger(__name__) + ROUTE_TOOL_NAME = "reportRoute" ROUTE_TOOL_DESCRIPTION = ( "提交只读项目分析的最终结果。必须在完成分析后调用一次，" "参数严格遵循给定的 JSON Schema；被拒绝时按返回的错误修正后重新调用。" ) +# What Codex is told when nobody answered the questions in time. +UNANSWERED_HINT = ( + "用户没有在时限内回答这些问题。请立即调用 " + f"{ROUTE_TOOL_NAME} 并返回 status=needs_input，" + "把原始问题写入 questions（每项包含 id、prompt、required），不要重复提问。" +) + _APP_SERVER_ENV = "AGENTKIT_MIGRATION_APP_SERVER" _DISABLED_VALUES = {"0", "false", "no", "off"} +# Publishes the questions to the page and returns the user's answers, or ``None`` when +# the user did not answer inside the window the caller allows. +AnalysisQuestioner = Callable[ + [tuple[dict[str, object], ...]], + Awaitable[Answers | None], +] + def app_server_analysis_enabled() -> bool: """Whether route analysis should use the Sandbox Codex app-server. @@ -87,6 +117,53 @@ def submit(self, arguments: dict[str, object]) -> CodexDynamicToolResult: ) +def ask_tool_handler(questioner: AnalysisQuestioner) -> Callable[..., object]: + """Build the ``askUser`` handler that bridges one turn to the browser. + + The result carries the answers exactly like Codex' native + ``ToolRequestUserInputResponse``, and an unanswered question set is a *successful* + call: the model is told nobody answered so it can fall back to ``needs_input`` + instead of asking again. The description lives on the tool spec, so a caller that + is not a read-only analysis can promise the right thing while reusing the channel. + """ + + async def handle(arguments: dict[str, object]) -> CodexDynamicToolResult: + try: + questions = normalize_questions(arguments) + except AnalysisAskError as error: + return CodexDynamicToolResult( + False, + f"提问不符合要求（{error}）。请修正后重新调用 {ASK_TOOL_NAME}。", + ) + try: + answers = await questioner(questions) + except Exception: # noqa: BLE001 - a broken channel must not kill the turn + logger.exception("Studio migration askUser channel failed") + answers = None + if answers is None: + return CodexDynamicToolResult( + True, + json.dumps( + {"answers": {}, "unanswered": True, "hint": UNANSWERED_HINT}, + ensure_ascii=False, + ), + ) + return CodexDynamicToolResult( + True, + json.dumps( + { + "answers": { + question_id: {"answers": list(values)} + for question_id, values in answers.items() + } + }, + ensure_ascii=False, + ), + ) + + return handle + + async def run_route_analysis( *, endpoint: str, @@ -98,9 +175,28 @@ async def run_route_analysis( model: str = "", timeout_seconds: float, event_sink: Callable[[object], None] | None = None, + questioner: AnalysisQuestioner | None = None, + idle_timeout_seconds: float | None = None, + host_wait_seconds: Callable[[], float] | None = None, ) -> dict[str, object] | None: - """Run one analysis turn and return the validated route contract, if any.""" + """Run one analysis turn and return the validated route contract, if any. + + Passing ``questioner`` registers ``askUser`` for this turn; the caller also owns + the matching window through ``idle_timeout_seconds`` and ``host_wait_seconds``. + """ recorder = RouteRecorder(attempt=attempt, input_sha256=input_sha256) + extra_tools = ( + ( + DynamicTool( + name=ASK_TOOL_NAME, + description=ASK_TOOL_DESCRIPTION, + schema=ASK_TOOL_SCHEMA, + handler=ask_tool_handler(questioner), + ), + ) + if questioner is not None + else () + ) try: await run_tool_turn( endpoint=endpoint, @@ -114,6 +210,9 @@ async def run_route_analysis( model=model, timeout_seconds=timeout_seconds, event_sink=event_sink, + extra_tools=extra_tools, + idle_timeout_seconds=idle_timeout_seconds, + host_wait_seconds=host_wait_seconds, ) except ToolTurnUnavailable as error: raise MigrationAnalysisUnavailable(str(error)) from error @@ -121,10 +220,12 @@ async def run_route_analysis( __all__ = [ + "AnalysisQuestioner", "MigrationAnalysisUnavailable", "ROUTE_TOOL_DESCRIPTION", "ROUTE_TOOL_NAME", "RouteRecorder", "app_server_analysis_enabled", + "ask_tool_handler", "run_route_analysis", ] diff --git a/frontend/server/migration/codex_tool_turn.py b/frontend/server/migration/codex_tool_turn.py index 1dac0811f..2df08d0e9 100644 --- a/frontend/server/migration/codex_tool_turn.py +++ b/frontend/server/migration/codex_tool_turn.py @@ -14,17 +14,19 @@ """One Codex app-server turn that reports its result through a dynamic tool. -Studio drives every structured Codex turn the same way: a dynamic tool registered on -``thread/start`` carries the result as typed JSON-RPC arguments, a rejected call returns +Studio drives every structured Codex turn the same way: dynamic tools registered on +``thread/start`` carry the result as typed JSON-RPC arguments, a rejected call returns ``success: false`` so the same turn can correct itself, and the turn ends as soon as the -contract has been delivered. Callers own the contract: they pass the tool spec, the -validator, and the check that says the result has arrived. +contract has been delivered. Callers own the contract: they pass the tool specs, the +validators, and the check that says the result has arrived. A turn may register several +tools, because one analysis can both ask the user a question and report its result. """ from __future__ import annotations import asyncio -from collections.abc import Callable +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass from veadk.cli.codex_app_server import ( CodexAppServerError, @@ -32,9 +34,13 @@ CodexDynamicToolResult, ) -ToolHandler = Callable[[dict[str, object]], CodexDynamicToolResult] +ToolHandler = Callable[ + [dict[str, object]], + "CodexDynamicToolResult | Awaitable[CodexDynamicToolResult]", +] __all__ = [ + "DynamicTool", "ToolHandler", "ToolTurnDeadlineExceeded", "ToolTurnUnavailable", @@ -54,6 +60,16 @@ class ToolTurnDeadlineExceeded(ToolTurnUnavailable): """ +@dataclass(frozen=True) +class DynamicTool: + """One host-provided tool that Codex may call inside the turn.""" + + name: str + description: str + schema: dict[str, object] + handler: ToolHandler + + async def run_tool_turn( *, endpoint: str, @@ -68,14 +84,25 @@ async def run_tool_turn( model: str = "", timeout_seconds: float, event_sink: Callable[[object], None] | None = None, + extra_tools: Sequence[DynamicTool] = (), + idle_timeout_seconds: float | None = None, + host_wait_seconds: Callable[[], float] | None = None, ) -> str: """Run one turn and return the thread id that carried it. ``thread_id`` resumes a durable thread instead of starting a new one; the app-server restores that thread's dynamic tools, so the same contract keeps - arriving. ``timeout_seconds`` is a wall-clock budget, not just the app-server's - inactivity window: callers that must answer inside a caller-owned window cannot - rely on a turn that keeps making progress, so this interrupts it at the deadline. + arriving. ``timeout_seconds`` is a wall-clock budget for Codex' own work, not just + the app-server's inactivity window: callers that must answer inside a caller-owned + window cannot rely on a turn that keeps making progress, so this interrupts it at + the deadline. ``host_wait_seconds`` excludes time a tool handler spent waiting on + a human, which is host latency and not Codex progress, from that budget. + + ``idle_timeout_seconds`` is the inactivity window handed to the app-server client. + A handler that blocks on a human produces no events at all while it waits, so a + caller with an interactive tool must pass an window longer than the longest wait it + allows, or the turn is cancelled under the waiting user. + Any protocol or transport failure becomes ``ToolTurnUnavailable`` so the caller can fall back, unless the result already arrived. """ @@ -83,13 +110,23 @@ async def run_tool_turn( session.cwd = cwd if model: session.model = model - session.register_dynamic_tool( - tool_name, - tool_description, - tool_schema, - handler, - ) + for tool in ( + DynamicTool( + name=tool_name, + description=tool_description, + schema=tool_schema, + handler=handler, + ), + *extra_tools, + ): + session.register_dynamic_tool( + tool.name, + tool.description, + tool.schema, + tool.handler, + ) used_thread = thread_id + loop = asyncio.get_running_loop() try: try: if thread_id: @@ -98,11 +135,15 @@ async def run_tool_turn( await session.connect() except CodexAppServerError as error: raise ToolTurnUnavailable(str(error)) from error - deadline = asyncio.get_running_loop().time() + timeout_seconds + started_at = loop.time() + deadline = started_at + timeout_seconds + idle_timeout = ( + timeout_seconds if idle_timeout_seconds is None else idle_timeout_seconds + ) try: async for event in session.stream_turn( prompt, - timeout_seconds=timeout_seconds, + timeout_seconds=idle_timeout, ): if event_sink is not None: event_sink(event) @@ -110,7 +151,11 @@ async def run_tool_turn( # 结果已经到手：终止本轮，避免继续消耗 token 和沙箱时间。 await session.interrupt() break - if asyncio.get_running_loop().time() >= deadline: + if host_wait_seconds is not None: + deadline = ( + started_at + timeout_seconds + max(0.0, host_wait_seconds()) + ) + if loop.time() >= deadline: # 回合一直在产生进度，但已经超出调用方的窗口：主动收尾。 await session.interrupt() raise ToolTurnDeadlineExceeded("Codex 回合超出时间预算。") diff --git a/frontend/server/migration/contracts.py b/frontend/server/migration/contracts.py index 93a9f6eb9..6d5b931de 100644 --- a/frontend/server/migration/contracts.py +++ b/frontend/server/migration/contracts.py @@ -367,6 +367,115 @@ def validate_process_exit(value: object) -> dict[str, object]: return {str(key): item for key, item in value.items()} +def validate_migration_driver( + value: object, + *, + expected_run_id: str, +) -> dict[str, object]: + """Validate the delivery driver lease, including the published artifact. + + The record is written inside the Sandbox by the launch script that supervises the + migration CLI. It lets Studio tell a driver that is still working from one whose + process disappeared, and it carries the artifact digest computed right after the + CLI exited, so the bytes Studio later pulls can be checked against it. + """ + if not isinstance(value, dict): + raise MigrationContractError("driver lease must be an object") + _exact_keys( + value, + required={ + "schema_version", + "run_id", + "state", + "heartbeat_at", + "finished_at", + "exit_code", + "artifact", + }, + ) + state = value.get("state") + if ( + value.get("schema_version") != 1 + or value.get("run_id") != expected_run_id + or state not in {"running", "finished"} + ): + raise MigrationContractError("invalid driver lease identity") + _bounded_integer(value.get("heartbeat_at"), maximum=10**12) + finished_at = value.get("finished_at") + exit_code = value.get("exit_code") + artifact = value.get("artifact") + if state == "running": + if finished_at is not None or exit_code is not None or artifact is not None: + raise MigrationContractError("running driver lease published a result") + else: + _bounded_integer(finished_at, maximum=10**12) + _bounded_integer(exit_code, maximum=255) + if artifact is not None: + if not isinstance(artifact, dict): + raise MigrationContractError("invalid artifact descriptor") + _exact_keys(artifact, required={"path", "sha256", "size"}) + if artifact.get("path") != "migration-result.zip": + raise MigrationContractError("invalid artifact path") + _bounded_integer(artifact.get("size"), maximum=_MAX_ARTIFACT_BYTES) + _sha256(artifact.get("sha256")) + return {str(key): item for key, item in value.items()} + + +def validate_delivery_report( + value: object, + *, + expected_run_id: str, + expected_state: str, +) -> dict[str, object]: + """Validate the verdict the closing delivery turn published. + + The turn explains a delivery; it never decides one. ``expected_state`` is the + state Studio derived from the Sandbox, and a report that disagrees with it is + rejected here as well as inside the turn, so a damaged or replayed record cannot + describe a delivery other than the one the CLI settled. + """ + if not isinstance(value, dict): + raise MigrationContractError("delivery report must be an object") + _exact_keys( + value, + required={ + "schema_version", + "run_id", + "driver", + "state", + "message", + "warnings", + "artifact", + "created_at", + }, + ) + state = value.get("state") + if ( + value.get("schema_version") != 1 + or value.get("run_id") != expected_run_id + or value.get("driver") != "app-server" + or state != expected_state + or state not in {"succeeded", "succeeded_with_warnings", "partial", "failed"} + ): + raise MigrationContractError("delivery report identity does not match") + _text(value.get("message"), allow_empty=False, maximum=4_000) + _string_list(value.get("warnings"), maximum_items=8) + artifact = value.get("artifact") + if state == "failed": + if artifact is not None: + raise MigrationContractError("failed delivery report published an artifact") + else: + if not isinstance(artifact, dict): + raise MigrationContractError("invalid delivery report artifact") + _exact_keys(artifact, required={"path", "sha256", "size"}) + if artifact.get("path") != "migration-result.zip": + raise MigrationContractError("invalid delivery report artifact path") + _bounded_integer(artifact.get("size"), maximum=_MAX_ARTIFACT_BYTES) + _sha256(artifact.get("sha256")) + _timestamp_text(value.get("created_at")) + return {str(key): item for key, item in value.items()} + + def validate_stopped_status(value: object) -> dict[str, object]: if not isinstance(value, dict): raise MigrationContractError("stopped status must be an object") @@ -812,7 +921,9 @@ def validate_delivery_result( "validate_analysis_status", "validate_confirmation", "validate_delivery_result", + "validate_delivery_report", "validate_delivery_status", + "validate_migration_driver", "validate_migration_request", "validate_process_exit", "validate_source_status", diff --git a/frontend/server/migration/delivery_turn.py b/frontend/server/migration/delivery_turn.py new file mode 100644 index 000000000..23e73bc79 --- /dev/null +++ b/frontend/server/migration/delivery_turn.py @@ -0,0 +1,401 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Close one migration delivery on a Studio-owned Codex app-server turn. + +The Sandbox runs the migration CLI and writes the delivery triple itself, so the +delivery phase used to end the moment the launch script published its status: a run +that stopped early reached the page as a generic "the command did not finish", and a +run that produced warnings reached it as one fixed sentence with the details left in a +log nobody opens. + +The closing turn keeps the AgentKit CLI as the authority on *what happened* and makes +Studio the authority on *what the user is told*: + +* the artifact is published through a dynamic tool, so Studio reads the bytes back and + checks them against the manifest the CLI wrote before the delivery may complete; +* the verdict arrives as typed arguments and is validated against the state Studio + derived from the Sandbox, so a turn that misreads a log can explain a delivery but + never upgrade or downgrade it; +* a failed run is diagnosed inside the turn, which can also ask for information only a + human has instead of leaving an unusable "see the logs" behind. + +``publishArtifact`` is why this turn has to live on the Studio side at all: dynamic +tools are registered on a Studio-driven app-server turn, and the Sandbox has no return +path to Studio, so an in-Sandbox ``codex exec`` can never call one. +""" + +from __future__ import annotations + +import os +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +from veadk.cli.codex_app_server import CodexDynamicToolResult + +from .codex_tool_turn import DynamicTool, ToolTurnUnavailable, run_tool_turn + +DELIVERY_TOOL_NAME = "reportDelivery" +DELIVERY_TOOL_DESCRIPTION = ( + "提交本次迁移交付的最终结论。必须调用一次，参数严格遵循给定的 JSON Schema；" + "被拒绝时按返回的错误修正后重新调用。" +) +ARTIFACT_TOOL_NAME = "publishArtifact" +ARTIFACT_TOOL_DESCRIPTION = ( + "发布本次迁移交付的产物文件，由 Studio 读取并核对字节。" + "参数 path 固定为 migration-result.zip。" +) + +# ``askUser`` inside the closing turn: the analysis wording promises a read-only +# analysis, and this turn is about a delivery that already happened. +DELIVERY_ASK_TOOL_DESCRIPTION = ( + "在交付收尾过程中向用户提出必须由用户决定的问题，并等待用户回答。" + "只在交付结论依赖用户才知道的信息时提问（例如缺失的密钥、部署目标、" + "是否接受降级交付），一次提出 1-3 个问题，每个问题给出简短 header 和完整 question；" + "有自然选择时给出 2-3 个 options（每个含 label 和 description，第一项为推荐项）。" +) + +# The states the AgentKit CLI can settle a delivery in. The turn may only report the +# one Studio derived from the Sandbox evidence. +DELIVERY_STATES = frozenset( + {"succeeded", "succeeded_with_warnings", "partial", "failed"} +) +ARTIFACT_PATH = "migration-result.zip" +_MAX_MESSAGE_CHARS = 600 +_MIN_MESSAGE_CHARS = 4 +_MAX_WARNINGS = 8 +_MAX_WARNING_CHARS = 400 + + +DELIVERY_APP_SERVER_ENV = "AGENTKIT_MIGRATION_DELIVERY_APP_SERVER" +_DISABLED_VALUES = {"0", "false", "no", "off"} + + +def delivery_app_server_enabled() -> bool: + """Whether a finished delivery is closed on a Studio Codex app-server turn. + + The turn explains the delivery the AgentKit CLI already settled and publishes its + artifact, so it is additive: when the app-server is unreachable, or when this is + switched off, the task keeps the CLI's own delivery state. Set + ``AGENTKIT_MIGRATION_DELIVERY_APP_SERVER=0`` to pin that scripted behaviour. + """ + return ( + os.getenv(DELIVERY_APP_SERVER_ENV, "").strip().lower() not in _DISABLED_VALUES + ) + + +class DeliveryContractError(ValueError): + """One delivery turn payload did not satisfy the delivery contract.""" + + +class DeliveryTurnUnavailable(ToolTurnUnavailable): + """The delivery turn ended without a verdict Studio can publish.""" + + +@dataclass(frozen=True) +class PublishedArtifact: + """An artifact Studio read back from the Sandbox and checked against the manifest.""" + + path: str + sha256: str + size: int + + def public(self) -> dict[str, object]: + return {"path": self.path, "sha256": self.sha256, "size": self.size} + + +# Reads the artifact inside the Sandbox and returns its verified descriptor, or raises +# ``DeliveryContractError`` when the bytes disagree with what the CLI manifest says. +ArtifactPublisher = Callable[[str], PublishedArtifact] + + +def _exact_arguments( + arguments: dict[str, object], + *, + expected: set[str], + tool: str, +) -> None: + """Reject arguments the tool schema does not describe. + + A dynamic tool result is typed JSON-RPC, and an unexpected field means the model is + answering a different contract than the one Studio registered. + """ + unknown = sorted(set(arguments) - expected) + if unknown: + raise DeliveryContractError(f"{tool} 不接受参数 " + "、".join(unknown)) + + +def _bounded_text(value: object, field: str, limit: int, *, minimum: int = 1) -> str: + if not isinstance(value, str): + raise DeliveryContractError(f"{field} 必须是字符串") + text = value.strip() + if len(text) < minimum: + raise DeliveryContractError( + f"{field} 必须说明具体情况（至少 {minimum} 个字符）" + ) + return text[:limit] + + +def _warnings(value: object) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + raise DeliveryContractError("warnings 必须是字符串数组") + warnings: list[str] = [] + for entry in value[:_MAX_WARNINGS]: + if not isinstance(entry, str): + raise DeliveryContractError("warnings 必须是字符串数组") + text = entry.strip() + if text: + warnings.append(text[:_MAX_WARNING_CHARS]) + return warnings + + +class DeliveryRecorder: + """Validate and retain the artifact and the verdict of one delivery turn. + + ``expected_state`` is the state Studio derived from the Sandbox evidence, and any + other state is rejected: the turn explains the delivery, it does not decide it. + """ + + def __init__( + self, + *, + run_id: str, + expected_state: str, + publisher: ArtifactPublisher, + ) -> None: + self.run_id = run_id + self.expected_state = expected_state + self.artifact: PublishedArtifact | None = None + self.verdict: dict[str, object] | None = None + self.rejections: list[str] = [] + self._publisher = publisher + + @property + def ready(self) -> bool: + return self.verdict is not None + + def publish(self, arguments: dict[str, object]) -> CodexDynamicToolResult: + try: + _exact_arguments( + arguments, + expected={"path"}, + tool=ARTIFACT_TOOL_NAME, + ) + except DeliveryContractError as error: + self.rejections.append(str(error)) + return CodexDynamicToolResult(False, f"参数不符合协议（{error}）。") + path = arguments.get("path") + if path != ARTIFACT_PATH: + self.rejections.append("artifact path") + return CodexDynamicToolResult( + False, + f"产物路径必须是 {ARTIFACT_PATH}，而不是 {path!r}。", + ) + try: + artifact = self._publisher(ARTIFACT_PATH) + except DeliveryContractError as error: + self.rejections.append(str(error)) + return CodexDynamicToolResult(False, f"产物核对失败（{error}）。") + except Exception as error: # noqa: BLE001 - a Sandbox read must not kill the turn + self.rejections.append(type(error).__name__) + return CodexDynamicToolResult( + False, + "产物读取失败，请确认迁移产物已经生成后重新调用。", + ) + self.artifact = artifact + return CodexDynamicToolResult( + True, + f"产物已核对：path={artifact.path} sha256={artifact.sha256} " + f"size={artifact.size}。请继续调用 {DELIVERY_TOOL_NAME} 提交结论。", + ) + + def report(self, arguments: dict[str, object]) -> CodexDynamicToolResult: + try: + _exact_arguments( + arguments, + expected={"state", "message", "warnings"}, + tool=DELIVERY_TOOL_NAME, + ) + except DeliveryContractError as error: + self.rejections.append(str(error)) + return CodexDynamicToolResult(False, f"参数不符合协议（{error}）。") + state = arguments.get("state") + if state not in DELIVERY_STATES: + self.rejections.append("state") + return CodexDynamicToolResult( + False, + "state 必须是 " + "、".join(sorted(DELIVERY_STATES)) + " 之一。", + ) + if state != self.expected_state: + self.rejections.append("state mismatch") + return CodexDynamicToolResult( + False, + f"这次交付的 state 已经确定为 {self.expected_state}，" + "请按沙箱里的交付证据重新调用。", + ) + try: + message = _bounded_text( + arguments.get("message"), + "message", + _MAX_MESSAGE_CHARS, + minimum=_MIN_MESSAGE_CHARS, + ) + warnings = _warnings(arguments.get("warnings")) + except DeliveryContractError as error: + self.rejections.append(str(error)) + return CodexDynamicToolResult(False, f"结论不符合协议（{error}）。") + if state == "failed" and self.artifact is not None: + self.rejections.append("failed delivery published an artifact") + return CodexDynamicToolResult( + False, + "这次交付已经失败，不应该有产物；请确认后重新调用。", + ) + if state != "failed" and self.artifact is None: + self.rejections.append("missing artifact") + return CodexDynamicToolResult( + False, + f"请先调用 {ARTIFACT_TOOL_NAME} 发布产物，再提交结论。", + ) + self.verdict = {"state": state, "message": message, "warnings": warnings} + return CodexDynamicToolResult(True, "交付结论已接收，请结束本轮。") + + +async def run_delivery_turn( + *, + endpoint: str, + prompt: str, + cwd: str, + run_id: str, + expected_state: str, + publisher: ArtifactPublisher, + model: str = "", + timeout_seconds: float, + event_sink: Callable[[object], None] | None = None, + extra_tools: Sequence[DynamicTool] = (), + idle_timeout_seconds: float | None = None, + host_wait_seconds: Callable[[], float] | None = None, +) -> dict[str, object] | None: + """Publish the artifact and the verdict of one finished delivery. + + Returns the validated report, or ``None`` when the turn ended without one; the + delivery state itself stays whatever the AgentKit CLI published. + """ + recorder = DeliveryRecorder( + run_id=run_id, + expected_state=expected_state, + publisher=publisher, + ) + try: + await run_tool_turn( + endpoint=endpoint, + prompt=prompt, + cwd=cwd, + tool_name=DELIVERY_TOOL_NAME, + tool_description=DELIVERY_TOOL_DESCRIPTION, + tool_schema=delivery_schema(), + handler=recorder.report, + has_result=lambda: recorder.ready, + model=model, + timeout_seconds=timeout_seconds, + event_sink=event_sink, + extra_tools=( + DynamicTool( + name=ARTIFACT_TOOL_NAME, + description=ARTIFACT_TOOL_DESCRIPTION, + schema=artifact_schema(), + handler=recorder.publish, + ), + *extra_tools, + ), + idle_timeout_seconds=idle_timeout_seconds, + host_wait_seconds=host_wait_seconds, + ) + except ToolTurnUnavailable as error: + # 任何回合故障都要变成调用方认识的那一种，否则它的兜底分支不会触发。 + raise DeliveryTurnUnavailable(str(error)) from error + if recorder.verdict is None: + return None + return { + "schema_version": 1, + "run_id": run_id, + "driver": "app-server", + **recorder.verdict, + "artifact": recorder.artifact.public() if recorder.artifact else None, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + + +def artifact_schema() -> dict[str, object]: + return { + "type": "object", + "additionalProperties": False, + "required": ["path"], + "properties": { + "path": { + "type": "string", + "description": f"产物文件名，固定为 {ARTIFACT_PATH}。", + } + }, + } + + +def delivery_schema() -> dict[str, object]: + return { + "type": "object", + "additionalProperties": False, + "required": ["state", "message", "warnings"], + "properties": { + "state": { + "type": "string", + "enum": sorted(DELIVERY_STATES), + "description": "本次交付的终态，必须与沙箱里的交付状态一致。", + }, + "message": { + "type": "string", + "description": ( + "给用户看的中文结论：成功时说明产物内容；失败时说明失败在哪一步、" + "关键证据是什么、用户下一步可以做什么。" + ), + }, + "warnings": { + "type": "array", + "items": {"type": "string"}, + "description": "用户需要知道的迁移提示，没有就留空数组。", + }, + }, + } + + +__all__ = [ + "ARTIFACT_PATH", + "DELIVERY_APP_SERVER_ENV", + "DELIVERY_ASK_TOOL_DESCRIPTION", + "ARTIFACT_TOOL_DESCRIPTION", + "ARTIFACT_TOOL_NAME", + "DELIVERY_STATES", + "DELIVERY_TOOL_DESCRIPTION", + "DELIVERY_TOOL_NAME", + "ArtifactPublisher", + "DeliveryContractError", + "DeliveryRecorder", + "DeliveryTurnUnavailable", + "PublishedArtifact", + "artifact_schema", + "delivery_app_server_enabled", + "delivery_schema", + "run_delivery_turn", +] diff --git a/frontend/server/migration/events.py b/frontend/server/migration/events.py new file mode 100644 index 000000000..696c08a8f --- /dev/null +++ b/frontend/server/migration/events.py @@ -0,0 +1,388 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Append-only event log that turns the migration page from polling into a stream. + +The Sandbox stays the source of truth. One poller per task runs the very same service +calls the page used to issue, and appends an event only when a payload actually changed; +subscribers replay from their cursor and then follow the log. A refresh, a second tab, +or a reconnecting client therefore resumes from a sequence number instead of driving its +own Sandbox reads, and progress no longer depends on a page being open. + +Payloads are whole snapshots rather than deltas: each event replaces the previous value +of its kind, so replaying a bounded window is always safe and a client that fell behind +the retained history simply re-syncs instead of corrupting its view. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import json +import time +from collections import deque +from collections.abc import AsyncIterator, Awaitable, Callable, Hashable +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import datetime, timezone + +from veadk.utils.logger import get_logger + +logger = get_logger(__name__) + +TASK_EVENT = "task" +ACTIVITY_EVENT = "activity" +ERROR_EVENT = "error" +DONE_EVENT = "done" + +DEFAULT_HISTORY_LIMIT = 256 +DEFAULT_POLL_SECONDS = 1.5 +DEFAULT_HEARTBEAT_SECONDS = 15.0 +DEFAULT_IDLE_SECONDS = 120.0 +DEFAULT_FAILURE_BACKOFF_SECONDS = 5.0 +DEFAULT_MAX_STREAMS = 64 + + +def _timestamp() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _digest(value: object) -> str: + serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True) +class MigrationEvent: + """One immutable log entry; ``seq`` is the cursor the client resumes from.""" + + seq: int + type: str + payload: dict[str, object] + + +@dataclass(frozen=True, slots=True) +class MigrationSnapshot: + """One read of a task, and whether it can still change without user action.""" + + task: dict[str, object] | None = None + activity: dict[str, object] | None = None + error: dict[str, object] | None = None + settled: bool = False + + +class MigrationEventLog: + """Monotonic, bounded, append-only log with a per-waiter wake-up.""" + + def __init__(self, *, history_limit: int = DEFAULT_HISTORY_LIMIT) -> None: + if history_limit < 1: + raise ValueError("history_limit must be positive") + self._events: deque[MigrationEvent] = deque(maxlen=history_limit) + self._seq = 0 + self._waiters: set[asyncio.Event] = set() + + @property + def last_seq(self) -> int: + return self._seq + + @property + def earliest_seq(self) -> int: + """First sequence still retained; one past the end while the log is empty.""" + return self._events[0].seq if self._events else self._seq + 1 + + @property + def size(self) -> int: + return len(self._events) + + def append(self, event_type: str, payload: dict[str, object]) -> MigrationEvent: + self._seq += 1 + event = MigrationEvent(self._seq, event_type, payload) + self._events.append(event) + waiters, self._waiters = self._waiters, set() + for waiter in waiters: + waiter.set() + return event + + def replay(self, after: int) -> list[MigrationEvent]: + """Retained events after ``after``; a stale cursor simply re-syncs.""" + return [event for event in self._events if event.seq > after] + + def wake(self) -> None: + """Release every waiter so it can re-read state that is not in the log.""" + waiters, self._waiters = self._waiters, set() + for waiter in waiters: + waiter.set() + + async def wait(self, after: int, timeout: float) -> bool: + """Wait for an event newer than ``after``; ``False`` when the wait timed out.""" + if self._seq > after: + return True + waiter = asyncio.Event() + self._waiters.add(waiter) + try: + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(waiter.wait(), timeout) + finally: + self._waiters.discard(waiter) + return self._seq > after + + +class MigrationStream: + """One poller and one log for a single task. + + The poller is the only writer, so sequence numbers, change detection, and the + ``done`` marker cannot race with a request handler. + """ + + def __init__( + self, + read: Callable[[], Awaitable[MigrationSnapshot]], + *, + poll_seconds: float = DEFAULT_POLL_SECONDS, + heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS, + idle_seconds: float = DEFAULT_IDLE_SECONDS, + failure_backoff_seconds: float = DEFAULT_FAILURE_BACKOFF_SECONDS, + history_limit: int = DEFAULT_HISTORY_LIMIT, + ) -> None: + self.log = MigrationEventLog(history_limit=history_limit) + self._read = read + self._poll_seconds = poll_seconds + self._heartbeat_seconds = heartbeat_seconds + self._idle_seconds = idle_seconds + self._failure_backoff_seconds = failure_backoff_seconds + self._digests: dict[str, str] = {} + self._subscribers = 0 + self._retire_at: float | None = None + self._settled = False + self._retired = False + self._runner: asyncio.Task[None] | None = None + self._failures = 0 + + @property + def settled(self) -> bool: + return self._settled + + @property + def subscribers(self) -> int: + return self._subscribers + + @property + def idle(self) -> bool: + """No subscriber is attached and the resume window has elapsed.""" + return ( + self._subscribers == 0 + and self._retire_at is not None + and time.monotonic() - self._retire_at >= self._idle_seconds + ) + + def attach(self) -> None: + self._subscribers += 1 + self._retire_at = None + if self._settled: + # Nothing left to poll: the follower replays the log and returns on ``done``. + return + if self._runner is None or self._runner.done(): + self._runner = asyncio.create_task(self._run()) + + def detach(self) -> None: + self._subscribers = max(0, self._subscribers - 1) + if self._subscribers == 0: + self._retire_at = time.monotonic() + + def retire(self) -> None: + """Stop polling and make every follower return. + + Retirement is not settlement: the task may still be running, the hub just has no + reason to keep a stream for it. A later subscriber builds a fresh stream. + """ + self._retired = True + runner = self._runner + if runner is not None and not runner.done(): + runner.cancel() + self.log.wake() + + async def follow(self, after: int) -> AsyncIterator[MigrationEvent | None]: + """Replay from ``after`` and then follow; ``None`` marks a heartbeat.""" + cursor = after + while True: + for event in self.log.replay(cursor): + cursor = event.seq + yield event + if event.type == DONE_EVENT: + return + if self._settled or self._retired: + return + if not await self.log.wait(cursor, self._heartbeat_seconds): + yield None + + async def _run(self) -> None: + try: + while True: + try: + snapshot = await self._read() + except asyncio.CancelledError: + raise + except Exception as error: # noqa: BLE001 - the page must survive a read fault + self._failures += 1 + if self._failures == 1 or self._failures % 10 == 0: + logger.warning( + "Studio migration event read failed attempts=%d error_type=%s", + self._failures, + type(error).__name__, + ) + self._publish( + MigrationSnapshot( + error={ + "code": "MIGRATION_EVENT_READ_FAILED", + "message": "迁移状态暂时无法读取，正在重试。", + "retryable": True, + } + ) + ) + if self.idle: + return + await asyncio.sleep(self._failure_backoff_seconds) + continue + self._failures = 0 + self._publish(snapshot) + if self._settled or self._retired: + return + if self.idle: + return + await asyncio.sleep(self._poll_seconds) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - a dead poller must not kill the loop silently + logger.exception("Studio migration event poller failed") + + def _publish(self, snapshot: MigrationSnapshot) -> None: + if snapshot.error is None: + # Clearing is a change too: a follower must drop a stale read-failure + # banner even when nothing else about the task moved. An ``error`` event + # without a code or a message is that "readable again" frame. + if self._digests.pop(ERROR_EVENT, None) is not None: + self.log.append(ERROR_EVENT, {}) + else: + self._append_changed(ERROR_EVENT, dict(snapshot.error)) + if snapshot.task is not None: + self._append_changed(TASK_EVENT, snapshot.task) + if snapshot.activity is not None: + self._append_changed(ACTIVITY_EVENT, snapshot.activity) + if snapshot.settled: + self._settle(snapshot.task) + + def _settle(self, task: dict[str, object] | None) -> None: + if self._settled: + return + self._settled = True + state = str(task.get("state") or "") if isinstance(task, dict) else "" + self.log.append(DONE_EVENT, {"state": state, "at": _timestamp()}) + + def _append_changed(self, event_type: str, payload: dict[str, object]) -> None: + digest = _digest(payload) + if self._digests.get(event_type) == digest: + return + self._digests[event_type] = digest + self.log.append(event_type, payload) + + +class MigrationEventHub: + """One stream per task, created on demand and dropped once nobody listens.""" + + def __init__( + self, + read: Callable[[Hashable], Callable[[], Awaitable[MigrationSnapshot]]], + *, + poll_seconds: float = DEFAULT_POLL_SECONDS, + heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS, + idle_seconds: float = DEFAULT_IDLE_SECONDS, + history_limit: int = DEFAULT_HISTORY_LIMIT, + max_streams: int = DEFAULT_MAX_STREAMS, + ) -> None: + self._read = read + self._poll_seconds = poll_seconds + self._heartbeat_seconds = heartbeat_seconds + self._idle_seconds = idle_seconds + self._history_limit = history_limit + self._max_streams = max(1, max_streams) + self._streams: dict[Hashable, MigrationStream] = {} + + def peek(self, key: Hashable) -> MigrationStream | None: + return self._streams.get(key) + + def stream(self, key: Hashable) -> MigrationStream: + """The live stream for ``key``, or a fresh one once it settled or went idle. + + A settled stream must never be handed out again: its log stops growing, so the + next subscriber would be told the task is over without anyone re-reading it. + """ + current = self._streams.get(key) + if current is not None and not current.idle and not current.settled: + return current + if current is not None: + current.retire() + self._trim(extra=1) + stream = MigrationStream( + self._read(key), + poll_seconds=self._poll_seconds, + heartbeat_seconds=self._heartbeat_seconds, + idle_seconds=self._idle_seconds, + history_limit=self._history_limit, + ) + self._streams[key] = stream + return stream + + @asynccontextmanager + async def subscription(self, key: Hashable) -> AsyncIterator[MigrationStream]: + """Attach for the lifetime of the block, then drop the stream once idle.""" + stream = self.stream(key) + stream.attach() + try: + yield stream + finally: + stream.detach() + self._reap() + + def close(self) -> None: + """Stop every poller; used on application shutdown.""" + for stream in self._streams.values(): + stream.retire() + self._streams.clear() + + def _reap(self) -> None: + for key, stream in list(self._streams.items()): + if stream.idle: + self._streams.pop(key, None) + stream.retire() + + def _trim(self, *, extra: int) -> None: + while len(self._streams) + extra > self._max_streams: + idle = [key for key, stream in self._streams.items() if stream.idle] + candidates = idle or list(self._streams) + key = candidates[0] + self._streams.pop(key, None).retire() + + +__all__ = [ + "ACTIVITY_EVENT", + "DONE_EVENT", + "ERROR_EVENT", + "TASK_EVENT", + "MigrationEvent", + "MigrationEventHub", + "MigrationEventLog", + "MigrationSnapshot", + "MigrationStream", +] diff --git a/frontend/server/migration/models.py b/frontend/server/migration/models.py index 3e4417cdd..fc0edc3ae 100644 --- a/frontend/server/migration/models.py +++ b/frontend/server/migration/models.py @@ -22,6 +22,7 @@ from pydantic import BaseModel, Field, model_validator +from .analysis_input import MAX_ANSWER_LENGTH, MAX_QUESTIONS from .evaluation.models import MigrationEvaluationConfig MigrationFramework = Literal[ @@ -169,6 +170,38 @@ def normalize(self) -> SubmitAnalysisAnswersBody: return self +class SubmitAnalysisInputBody(BaseModel): + """Answers for the questions a running analysis turn asked the user.""" + + request_id: str = Field(alias="requestId", min_length=1, max_length=64) + answers: dict[str, str] + + model_config = {"populate_by_name": True, "extra": "forbid"} + + @model_validator(mode="after") + def normalize(self) -> SubmitAnalysisInputBody: + self.request_id = self.request_id.strip() + if not self.request_id: + raise ValueError("分析问题标识无效") + if not self.answers: + raise ValueError("请先回答全部分析问题") + if len(self.answers) > MAX_QUESTIONS: + raise ValueError(f"分析问题不能超过 {MAX_QUESTIONS} 个") + normalized_answers: dict[str, str] = {} + for key, value in self.answers.items(): + normalized_key = key.strip() + normalized_value = value.strip() + if not normalized_key or len(normalized_key) > 64: + raise ValueError("分析问题 ID 无效") + if not normalized_value: + raise ValueError("分析问题的回答不能为空") + if len(normalized_value) > MAX_ANSWER_LENGTH: + raise ValueError(f"单个分析回答不能超过 {MAX_ANSWER_LENGTH} 个字符") + normalized_answers[normalized_key] = normalized_value + self.answers = normalized_answers + return self + + __all__ = [ "MIGRATION_FRAMEWORKS", "STRUCTURED_ENTRY_PATTERN", @@ -177,5 +210,6 @@ def normalize(self) -> SubmitAnalysisAnswersBody: "CreateMigrationTaskBody", "MigrationFramework", "SubmitAnalysisAnswersBody", + "SubmitAnalysisInputBody", "is_valid_structured_entry", ] diff --git a/frontend/server/migration/routes.py b/frontend/server/migration/routes.py index bc31b23eb..f1ddefd91 100644 --- a/frontend/server/migration/routes.py +++ b/frontend/server/migration/routes.py @@ -17,20 +17,23 @@ from __future__ import annotations import asyncio +import json import logging -from collections.abc import Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any from fastapi import HTTPException, Query, Request from fastapi.concurrency import run_in_threadpool -from fastapi.responses import Response +from fastapi.responses import Response, StreamingResponse from .evaluation.models import EvaluationDatasetBody, ResumeEvaluationBody from .evaluation.service import MigrationEvaluationService +from .events import MigrationEventHub, MigrationSnapshot from .models import ( ConfirmMigrationBody, CreateMigrationTaskBody, SubmitAnalysisAnswersBody, + SubmitAnalysisInputBody, ) from .service import ( MIGRATION_UPLOAD_MAX_BYTES, @@ -42,11 +45,54 @@ if TYPE_CHECKING: from frontend.server.source_projects import SourceProjectService + +def migration_activity_visible(task: dict[str, object]) -> bool: + """Whether the page shows the Codex activity feed for this task.""" + if task.get("state") == "analyzing": + return True + if task.get("analysisRef") or task.get("confirmation"): + return True + error = task.get("error") + code = str(error.get("code") or "") if isinstance(error, dict) else "" + return code.startswith("MIGRATION_ANALYSIS_") + + +def migration_task_settled(task: dict[str, object]) -> bool: + """Whether the task stops changing until the user acts on it. + + The page used to decide this on its own and stop polling; owning the rule here lets + the stream close at the same moment instead of polling a parked task. + """ + if str(task.get("state") or "") in _ACTIVE_TASK_STATES: + return False + persistence = task.get("persistence") + if isinstance(persistence, dict) and persistence.get("state") == "saving": + return False + evaluation = task.get("evaluation") + if ( + isinstance(evaluation, dict) + and evaluation.get("enabled") is True + and str(evaluation.get("state") or "") in _POLLING_EVALUATION_STATES + ): + return False + return True + + _ZIP_CONTENT_TYPES = { "application/zip", "application/x-zip-compressed", "application/octet-stream", } +# 这两组状态决定事件流什么时候可以收尾；页面以前自己复制了一份，现在由服务端拥有。 +_ACTIVE_TASK_STATES = {"analyzing", "migrating", "validating", "packaging"} +_POLLING_EVALUATION_STATES = { + "pending", + "preparing", + "deploying", + "executing", + "judging", + "aggregating", +} def mount_migration_routes( @@ -62,6 +108,60 @@ def mount_migration_routes( persistence_tasks: dict[tuple[str, str], asyncio.Task[dict[str, object]]] = {} watchers: dict[tuple[str, str], asyncio.Task[None]] = {} + def migration_stream_reader( + key: tuple[str, str], + ) -> Callable[[], Awaitable[MigrationSnapshot]]: + """Read the same payloads the page used to poll for, once per stream tick.""" + owner_id, task_id = key + + async def read() -> MigrationSnapshot: + try: + task = await run_in_threadpool(service.get_task, task_id, owner_id) + # A background app-server analysis dies with the Studio process; the + # stream owns this recovery now that the page no longer polls. + await run_in_threadpool( + service.recover_stalled_analysis, + task_id, + owner_id, + ) + # 交付收尾回合同样活在 Studio 进程里：读任务的这条路径负责在交付落定后 + # 补一次收尾，Studio 重启丢掉的那一轮也能在这里接回来。 + await run_in_threadpool( + service.drive_delivery_turn, + task_id, + owner_id, + task=task, + ) + decorated = await with_evaluation(task, owner_id) + activity = None + if migration_activity_visible(decorated): + activity = await run_in_threadpool( + service.activity, + task_id, + owner_id, + ) + settled = migration_task_settled(decorated) + evaluation = decorated.get("evaluation") + if ( + not settled + and isinstance(evaluation, dict) + and evaluation.get("enabled") is True + ): + start_watcher(task_id, owner_id) + return MigrationSnapshot( + task=decorated, + activity=activity, + settled=settled, + ) + except MigrationError as error: + if error.retryable: + raise + return MigrationSnapshot(error=error.detail(), settled=True) + + return read + + event_hub = MigrationEventHub(migration_stream_reader) + async def invoke( operation: str, call: Callable[[], Any], @@ -316,6 +416,14 @@ async def watch() -> None: continue return state = task.get("state") + # 交付落定后由 Studio 收尾一次，所以看护循环也要给收尾回合机会， + # 不能只在页面打开的时候才收尾。 + await run_in_threadpool( + service.drive_delivery_turn, + task_id, + owner_id, + task=task, + ) if state in { "succeeded", "succeeded_with_warnings", @@ -510,6 +618,13 @@ async def get_task( lambda: service.recover_stalled_analysis(task_id, owner_id), task_id=task_id, ) + # 交付落定以后由 Studio 收尾一次：成功时核对并发布产物，失败时把日志读成 + # 一句能解释的结论。回合跑在后台线程里，所以这次调用只做一次廉价判断。 + await invoke( + "drive_delivery_turn", + lambda: service.drive_delivery_turn(task_id, owner_id, task=task), + task_id=task_id, + ) decorated = await with_evaluation(task, owner_id) evaluation = decorated.get("evaluation") if isinstance(evaluation, dict) and evaluation.get("enabled") is True: @@ -530,6 +645,20 @@ async def submit_answers( ) return await with_evaluation(task, owner_id) + @app.post("/web/agent-migrations/tasks/{task_id}/input") + async def submit_analysis_input( + task_id: str, + body: SubmitAnalysisInputBody, + request: Request, + ) -> dict[str, object]: + owner_id = owner_resolver(request) + task = await invoke( + "submit_analysis_input", + lambda: service.submit_analysis_input(task_id, owner_id, body), + task_id=task_id, + ) + return await with_evaluation(task, owner_id) + @app.post("/web/agent-migrations/tasks/{task_id}/confirm") async def confirm( task_id: str, @@ -740,6 +869,46 @@ async def activity( task_id=task_id, ) + @app.get("/web/agent-migrations/tasks/{task_id}/events") + async def task_events( + task_id: str, + request: Request, + after: int = Query(0, ge=0), + ) -> StreamingResponse: + owner_id = owner_resolver(request) + # Authorize before the streaming response commits its headers, so a stranger + # gets a normal error instead of an empty 200 stream. + await invoke( + "get_task", + lambda: service.get_task(task_id, owner_id), + task_id=task_id, + ) + + async def frames() -> AsyncIterator[str]: + async with event_hub.subscription((owner_id, task_id)) as stream: + # A cursor from a stream that is gone (Studio restarted) would wait for + # events that will never come; the retained history re-syncs instead. + start = after if after <= stream.log.last_seq else 0 + async for event in stream.follow(start): + if event is None: + yield ": heartbeat\n\n" + continue + payload = { + **event.payload, + "seq": event.seq, + "taskId": task_id, + } + yield ( + f"id: {event.seq}\nevent: {event.type}\n" + f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + ) + + return StreamingResponse( + frames(), + media_type="text/event-stream", + headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"}, + ) + @app.get("/web/agent-migrations/tasks/{task_id}/artifact") async def artifact( task_id: str, diff --git a/frontend/server/migration/service.py b/frontend/server/migration/service.py index 7cd316355..1262482f9 100644 --- a/frontend/server/migration/service.py +++ b/frontend/server/migration/service.py @@ -56,8 +56,10 @@ validate_analysis_result, validate_analysis_status, validate_confirmation, + validate_delivery_report, validate_delivery_result, validate_delivery_status, + validate_migration_driver, validate_migration_request, validate_process_exit, validate_source_status, @@ -71,11 +73,33 @@ MigrationRemoteFileNotFound, MigrationSandboxSession, ) +from .activity import AnalysisActivityLog +from .analysis_input import ( + ASK_TOOL_NAME, + ASK_TOOL_SCHEMA, + AnalysisAskError, + AnalysisInputRegistry, + ask_payload, + normalize_answers, +) from .app_server import ( MigrationAnalysisUnavailable, app_server_analysis_enabled, + ask_tool_handler, run_route_analysis, ) +from .codex_tool_turn import DynamicTool +from .delivery_turn import ( + ARTIFACT_PATH, + ARTIFACT_TOOL_NAME, + DELIVERY_ASK_TOOL_DESCRIPTION, + DELIVERY_TOOL_NAME, + DeliveryContractError, + DeliveryTurnUnavailable, + PublishedArtifact, + delivery_app_server_enabled, + run_delivery_turn, +) from .models import ( MIGRATION_FRAMEWORKS, STRUCTURED_ENTRY_PATTERN, @@ -83,6 +107,7 @@ ConfirmMigrationBody, CreateMigrationTaskBody, SubmitAnalysisAnswersBody, + SubmitAnalysisInputBody, ) MIGRATION_ROOT = "/home/gem/.studio/migration/v1" @@ -134,6 +159,13 @@ _ANALYSIS_EXTRACTION_DIAGNOSTICS_PATH = ( f"{MIGRATION_ROOT}/diagnostics/analysis/result-extraction.json" ) + + +def _analysis_activity_path(attempt: int) -> str: + """Where an analysis attempt's Codex event log lives inside the Sandbox.""" + return f"{MIGRATION_ROOT}/diagnostics/analysis/attempt-{attempt}.log" + + _ANALYSIS_CONTRACT_KEYS = ( "schema_version", "status", @@ -150,6 +182,10 @@ ) _ANALYSIS_CONTRACT_STATUSES = ("needs_input", "recommendation_ready", "unsupported") _ANALYSIS_TURN_TIMEOUT_SECONDS = 600.0 +# 分析回合等待用户回答的窗口：等待期间没有 app-server 事件，所以空闲窗口必须 +# 覆盖它，否则客户端的空闲计时器会在用户作答前取消整个回合。 +_ANALYSIS_INPUT_WINDOW_SECONDS = 300.0 +_ANALYSIS_INPUT_IDLE_MARGIN_SECONDS = 60.0 _ANALYSIS_DRIVER_PATH = f"{MIGRATION_ROOT}/control/analysis-driver.json" _ANALYSIS_DRIVER_APP_SERVER = "app-server" _ANALYSIS_DRIVER_SCRIPT = "codex-exec" @@ -172,6 +208,39 @@ _DELIVERY_STATUS_PATH = f"{MIGRATION_ROOT}/delivery/migration-status.json" _DELIVERY_RESULT_PATH = f"{MIGRATION_ROOT}/delivery/migration-result.json" _DELIVERY_ARTIFACT_PATH = f"{MIGRATION_ROOT}/delivery/migration-result.zip" +# The delivery driver lease: the Sandbox launch script refreshes its heartbeat while +# the migration CLI works and publishes the artifact digest once it exits. Studio +# reads it to tell "still working" from "the process is gone", so a driver that died +# without writing any delivery state fails the task instead of hanging in migrating. +_MIGRATION_DRIVER_PATH = f"{MIGRATION_ROOT}/control/migration-driver.json" +_MIGRATION_DRIVER_SCRIPT_PATH = f"{MIGRATION_ROOT}/control/migration-driver.py" +_MIGRATION_DRIVER_HEARTBEAT_SECONDS = 15.0 +_MIGRATION_DRIVER_STALE_SECONDS = 90.0 +# 交付收尾回合：迁移 CLI 结束以后，Studio 驱动一个 app-server 回合核对并发布这次交付。 +# 产物由 Studio 自己读回、自己算摘要，失败也在这里变成一句能解释、能追问的结论。 +_DELIVERY_REPORT_PATH = f"{MIGRATION_ROOT}/delivery/delivery-report.json" +_DELIVERY_TURN_PATH = f"{MIGRATION_ROOT}/control/delivery-turn.json" +_DELIVERY_TURN_CWD = f"{MIGRATION_ROOT}/work/delivery" +_DELIVERY_TURN_ACTIVITY_PATH = f"{MIGRATION_ROOT}/work/agentic/logs/delivery-turn.jsonl" +_DELIVERY_TURN_DRIVER = "app-server" +_DELIVERY_TURN_RUNNING = "running" +_DELIVERY_TURN_DONE = "done" +_DELIVERY_TURN_TIMEOUT_SECONDS = 300.0 +# 等待用户回答期间没有 app-server 事件，空闲窗口必须覆盖它，否则回合会被取消。 +_DELIVERY_TURN_INPUT_WINDOW_SECONDS = 300.0 +_DELIVERY_TURN_INPUT_IDLE_MARGIN_SECONDS = 60.0 +# 收尾回合的租约：心跳过期说明持有它的 Studio 进程已经不在了，可以重新收尾。 +_DELIVERY_TURN_HEARTBEAT_SECONDS = 20.0 +_DELIVERY_TURN_STALE_SECONDS = 90.0 +# 一个交付最多收尾几次：失败后每次读任务都重开回合会白白烧 token。 +_DELIVERY_TURN_MAX_ATTEMPTS = 2 +# 交付阶段已经落定的状态：只有落定的交付才需要收尾回合解释它。 +_DELIVERY_SETTLED_STATES = { + "succeeded", + "succeeded_with_warnings", + "partial", + "failed", +} _MIGRATION_ACTIVITY_LOG_PATHS = tuple( f"{MIGRATION_ROOT}/work/agentic/logs/codex-attempt-{attempt}.jsonl" for attempt in range(1, 4) @@ -1370,6 +1439,24 @@ def _analysis_schema() -> dict[str, object]: } +def _interactive_analysis_context() -> str: + """The ask-by-tool section, only for turns that registered ``askUser``.""" + return """ +## 交互提问（本回合可用） + +- 本轮已注册 askUser 工具。当项目内容无法回答、且答案会改变迁移方式、入口或范围时， + 必须先用 askUser 直接询问用户，不要先把结论交付出去。 +- 一次提问 1-3 个问题，每个问题给出简短 header 和完整 question；有自然选择时给出 + 2-3 个 options（每项含 label 和 description，第一项为推荐项），没有自然选择时省略 options。 +- 用户回答会在同一次分析中返回。拿到回答后继续完成分析，并用 reportRoute 交付最终结论， + 不要重复提问已经问过的问题。 +- 只有 askUser 返回 unanswered（用户没有在时限内回答）时，才用 status=needs_input + 交付这些问题，让用户之后在页面上补充。 +- 能从项目文件确认的事实必须自己查证，禁止为了省事而提问。 + +""" + + def _analysis_prompt( request: dict[str, object], *, @@ -1378,8 +1465,11 @@ def _analysis_prompt( previous_analysis: dict[str, object] | None = None, answers: dict[str, str] | None = None, protocol_retry: bool = False, + interactive: bool = False, ) -> str: instruction = str(request.get("instruction") or "").strip() + # 只有 app-server 驱动注册了 askUser；脚本驱动读到的提示词不能承诺这个工具。 + interactive_context = _interactive_analysis_context() if interactive else "" retry_context = ( "\n## 协议重试\n" "上一次回复无法作为分析结果读取：其中没有符合输出协议的 JSON 对象。" @@ -1523,7 +1613,7 @@ def _analysis_prompt( 不要只输出错误码、框架术语或“未找到可执行方式”之类没有行动建议的表述。 - warnings 要具体描述缺失材料及影响，不得把可在迁移或部署阶段补齐的条件写成阻塞项。 -## 输出协议 +{interactive_context}## 输出协议 - 顶层字段必须且只能是：schema_version、status、attempt、input_sha256、 summary、frameworks、recommended、entries、boundary、assumptions、questions、warnings。 @@ -1852,6 +1942,83 @@ def _analysis_driver_marker( } +def _delivery_turn_marker( + *, + state: str, + attempts: int = 1, + verdict: bool | None = None, + started_at: float | None = None, + heartbeat_at: float | None = None, +) -> dict[str, object]: + """Describe the Studio worker that owns the closing delivery turn. + + ``verdict`` says whether the turn that ended published a report, so a delivery that + was already tried is not retried on every later read of the same task. + """ + started = time.time() if started_at is None else started_at + return { + "schema_version": 1, + "driver": _DELIVERY_TURN_DRIVER, + "state": state, + "attempts": attempts, + "verdict": verdict, + "started_at": started, + "heartbeat_at": started if heartbeat_at is None else heartbeat_at, + "owner_process": _STUDIO_PROCESS_ID, + } + + +def _delivery_prompt( + *, + task_id: str, + framework: str, + expected_state: str, + exit_code: object, +) -> str: + """The instructions for the turn that closes one finished delivery.""" + logs = f"{MIGRATION_ROOT}/work/agentic/logs" + exit_code_text = str(exit_code) if exit_code is not None else "未知" + return "\n".join( + [ + "# 迁移交付收尾", + "", + "沙箱里的迁移命令已经结束，现在由你核对这次迁移实际交付了什么，", + "并把结论发布给用户。交付状态由 AgentKit CLI 决定，你只负责解释它。", + "", + f"- 运行 ID：{task_id}", + f"- 迁移框架：{framework}（agentic）", + f"- 迁移命令退出码：{exit_code_text}", + f"- 这次交付的状态已经确定为：{expected_state}", + "", + "## 证据文件（沙箱内绝对路径）", + f"- 交付状态：`{_DELIVERY_STATUS_PATH}`", + f"- 产物清单：`{_DELIVERY_RESULT_PATH}`", + f"- 交付产物：`{_DELIVERY_ARTIFACT_PATH}`", + f"- 迁移任务日志：`{logs}/task.log`", + f"- 校验日志：`{logs}/validation-attempt-1.log`", + f"- Codex 事件流：`{logs}/codex-attempt-1.jsonl`", + f"- 迁移命令日志：`{MIGRATION_ROOT}/diagnostics/migration/migration.log`", + "", + "## 执行顺序", + "1. 读上面的证据文件，弄清这次交付的结果：产物包含什么、有哪些提示、", + " 如果是失败，失败发生在哪一步（Codex 尝试、校验、打包）。", + f"2. 调用 `{ARTIFACT_TOOL_NAME}`，参数 `path` 固定为 `{ARTIFACT_PATH}`，", + " 由 Studio 读取并核对产物字节。交付失败时跳过这一步。", + f"3. 调用 `{DELIVERY_TOOL_NAME}` 提交结论，参数严格按给定 Schema：", + f" - `state` 必须等于 {expected_state}，其它取值会被拒绝；", + " - `message` 是给用户看的一句中文结论：成功时说清产物内容，", + " 失败时说清失败在哪一步、日志里的关键证据、用户下一步可以做什么；", + " - `warnings` 是用户需要知道的迁移提示，没有就留空数组。", + "", + "## 约束", + "- 不要修改产物、不要重跑迁移、不要执行会改变沙箱状态的命令。", + "- 失败原因如果只有用户能提供（例如缺失的模型密钥、部署目标、是否接受降级），", + " 可以调用 `askUser` 提问，然后按回答给出结论。", + "- 不要输出 Markdown 表格，不要贴大段日志原文。", + ] + ) + + def _start_analysis_command(task_id: str, attempt: int) -> str: running_status = _analysis_running_status(attempt) ready_status = { @@ -1911,7 +2078,7 @@ def _start_analysis_command(task_id: str, attempt: int) -> str: }, } result_tmp = f"{_ANALYSIS_RESULT_PATH}.{attempt}.tmp" - log_path = f"{MIGRATION_ROOT}/diagnostics/analysis/attempt-{attempt}.log" + log_path = _analysis_activity_path(attempt) pid_path = f"{MIGRATION_ROOT}/control/analysis.pid" lock_path = f"{MIGRATION_ROOT}/control/analysis-start-{attempt}.lock" validate_json = shlex.quote( @@ -2180,6 +2347,89 @@ def _ak_command( return " ".join(shlex.quote(item) for item in common) +_MIGRATION_DRIVER_TEMPLATE = '''"""Publish the migration driver lease and the artifact manifest. + +Written into the Sandbox by the launch script and run twice: in the background to +keep the heartbeat fresh while the migration CLI works, and once after it exits to +publish the finished record with the artifact digest. +""" + +import hashlib +import json +import os +import sys +import time +from pathlib import Path + +__HEARTBEAT_SECONDS__ + +path = Path(sys.argv[1]) +artifact = Path(sys.argv[2]) +run_id = sys.argv[3] +mode = sys.argv[4] + + +def publish(value): + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + os.replace(temporary, path) + + +def lease(state, heartbeat_at, finished_at=None, exit_code=None, artifact_entry=None): + return { + "schema_version": 1, + "run_id": run_id, + "state": state, + "heartbeat_at": heartbeat_at, + "finished_at": finished_at, + "exit_code": exit_code, + "artifact": artifact_entry, + } + + +def manifest(): + """Return the artifact descriptor, or None when the CLI produced no archive.""" + if not artifact.is_file(): + return None + digest = hashlib.sha256() + size = 0 + with artifact.open("rb") as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + digest.update(chunk) + return {"path": artifact.name, "sha256": digest.hexdigest(), "size": size} + + +if mode == "heartbeat": + publish(lease("running", int(time.time()))) + while True: + time.sleep(HEARTBEAT_SECONDS) + publish(lease("running", int(time.time()))) +else: + now = int(time.time()) + publish( + lease( + "finished", + now, + finished_at=now, + exit_code=int(sys.argv[5]), + artifact_entry=manifest(), + ) + ) +''' + + +def _migration_driver_script() -> str: + """The Sandbox-side script that publishes the delivery driver lease.""" + return _MIGRATION_DRIVER_TEMPLATE.replace( + "__HEARTBEAT_SECONDS__", + f"HEARTBEAT_SECONDS = {_MIGRATION_DRIVER_HEARTBEAT_SECONDS}", + ) + + def _start_migration_command( task_id: str, confirmation: dict[str, object], @@ -2222,9 +2472,26 @@ def _start_migration_command( ), ] ) + driver = " ".join( + [ + "python3", + shlex.quote(_MIGRATION_DRIVER_SCRIPT_PATH), + shlex.quote(_MIGRATION_DRIVER_PATH), + shlex.quote(_DELIVERY_ARTIFACT_PATH), + shlex.quote(task_id), + ] + ) inner = "\n".join( [ "set +e", + ( + f"cat > {shlex.quote(_MIGRATION_DRIVER_SCRIPT_PATH)} " + "<<'STUDIO_MIGRATION_DRIVER'" + ), + _migration_driver_script(), + "STUDIO_MIGRATION_DRIVER", + f"{driver} heartbeat &", + "driver_pid=$!", "(", "set -e", *validation_model_env, @@ -2232,6 +2499,9 @@ def _start_migration_command( cli, f") > {shlex.quote(log_path)} 2>&1", "code=$?", + 'kill "$driver_pid" 2>/dev/null', + 'wait "$driver_pid" 2>/dev/null', + f'{driver} finish "$code"', "finished_at=$(python3 -c 'import time; print(int(time.time()))')", ( f'printf \'%s\\n\' "{{\\"schema_version\\":1,' @@ -2378,6 +2648,10 @@ def __init__( self._clock = clock # 进程内的后台分析驱动，键为 (session_id, attempt)，避免重复起同一轮分析。 self._analysis_drivers: dict[tuple[str, int], threading.Thread] = {} + # 正在等待用户回答的分析提问，由 HTTP 线程投递答案。 + self._analysis_input = AnalysisInputRegistry() + # 进程内的交付收尾回合，键为 session_id，避免同一交付重复收尾。 + self._delivery_turns: dict[str, threading.Thread] = {} @staticmethod def _translate(error: MigrationGatewayError) -> MigrationError: @@ -2854,6 +3128,60 @@ def _process_exit_is_settling(self, process_exit: dict[str, object]) -> bool: age = self._clock() - finished_at return -_REMOTE_CLOCK_SKEW_SECONDS <= age < _REMOTE_STATE_SETTLE_SECONDS + def _read_migration_driver( + self, + session: MigrationSandboxSession, + ) -> dict[str, object] | None: + """Read the delivery driver lease, tolerating a missing or damaged record. + + The lease is control-plane bookkeeping rather than a delivery contract, so a + record that cannot be read or validated is reported and ignored instead of + making every later read of the task fail. + """ + try: + driver = self._read_json( + session, + _MIGRATION_DRIVER_PATH, + optional=True, + ) + except MigrationError: + logger.warning( + "Studio migration driver lease is unreadable task_id=%s", + session.task_id, + ) + return None + if driver is None: + return None + try: + return validate_migration_driver( + driver, + expected_run_id=session.task_id, + ) + except MigrationContractError as error: + logger.warning( + "Studio migration driver lease is invalid task_id=%s error=%s", + session.task_id, + error, + ) + return None + + def _migration_driver_lost( + self, + driver: dict[str, object] | None, + ) -> bool: + """Whether a running delivery driver stopped reporting in. + + The Sandbox runs the AgentKit CLI and its heartbeat in one process group, so a + heartbeat that stops advancing means that run is gone and nothing will ever + write the delivery state the task is waiting for. + """ + if not isinstance(driver, dict) or driver.get("state") != "running": + return False + heartbeat = driver.get("heartbeat_at") + if isinstance(heartbeat, bool) or not isinstance(heartbeat, int): + return False + return self._clock() - float(heartbeat) >= _MIGRATION_DRIVER_STALE_SECONDS + @staticmethod def _validate_request( existing: dict[str, object], @@ -2967,6 +3295,7 @@ def upload_source( request, attempt=1, input_sha256=digest, + interactive=True, ), attempt=1, input_sha256=digest, @@ -3112,6 +3441,15 @@ def _app_server_analysis_worker( ) analysis = None if analysis is None: + # The turn can also end without ever delivering the contract, which is + # why the driver switches here as well; say so, or an operator only + # sees a scripted log with no explanation of where it came from. + logger.warning( + "Studio migration app-server analysis returned no result; " + "continuing with the scripted driver task_id=%s attempt=%s", + session.task_id, + attempt, + ) self._start_scripted_analysis( session, task_id=session.task_id, @@ -3176,7 +3514,35 @@ async def beat() -> None: type(error).__name__, ) + # The scripted driver's activity log is written inside the Sandbox by + # ``codex exec --json``; an app-server turn only exists on the wire, so its + # events are recorded into the very same file. One reader then serves both + # drivers, and the page shows what Codex is doing on either path. + activity = AnalysisActivityLog( + lambda content: self._put( + session, + _analysis_activity_path(attempt), + content, + media_type="text/plain", + ) + ) + # 用户在回合内作答的时间不算 Codex 的工作时间，从墙钟预算里扣除。 + waited_seconds = [0.0] + + async def questioner( + questions: tuple[dict[str, object], ...], + ) -> dict[str, tuple[str, ...]] | None: + """Publish one question set and wait for the page to answer it.""" + return await self._ask_user( + session, + questions=questions, + attempt=attempt, + window_seconds=_ANALYSIS_INPUT_WINDOW_SECONDS, + waited_seconds=waited_seconds, + ) + heartbeat = asyncio.create_task(beat()) + flusher = asyncio.create_task(activity.run()) try: return await run_route_analysis( endpoint=session.endpoint, @@ -3187,8 +3553,20 @@ async def beat() -> None: input_sha256=input_sha256, model=model_id, timeout_seconds=timeout_seconds, + event_sink=activity.record, + questioner=questioner, + idle_timeout_seconds=( + timeout_seconds + + _ANALYSIS_INPUT_WINDOW_SECONDS + + _ANALYSIS_INPUT_IDLE_MARGIN_SECONDS + ), + host_wait_seconds=lambda: waited_seconds[0], ) finally: + flusher.cancel() + with contextlib.suppress(asyncio.CancelledError): + await flusher + await activity.aclose() heartbeat.cancel() with contextlib.suppress(asyncio.CancelledError): await heartbeat @@ -3322,6 +3700,549 @@ def recover_stalled_analysis(self, task_id: str, owner_id: str) -> bool: return False return True + async def _ask_user( + self, + session: MigrationSandboxSession, + *, + questions: tuple[dict[str, object], ...], + attempt: int, + window_seconds: float, + waited_seconds: list[float], + ) -> dict[str, tuple[str, ...]] | None: + """Publish one question set and wait for the page to answer it. + + The wait is host latency rather than Codex progress, so callers pass the + accumulator their turn uses to keep that time out of its own budget. + """ + pending = self._analysis_input.open( + session.session_id, + attempt=attempt, + questions=questions, + ) + loop = asyncio.get_running_loop() + started = loop.time() + try: + answers = await asyncio.to_thread( + pending.future.result, + window_seconds, + ) + except TimeoutError: + logger.info( + "Studio migration question timed out task_id=%s attempt=%s " + "window_seconds=%s", + session.task_id, + attempt, + window_seconds, + ) + answers = None + finally: + waited_seconds[0] += loop.time() - started + self._analysis_input.discard( + session.session_id, + request_id=pending.request_id, + ) + return answers + + def drive_delivery_turn( + self, + task_id: str, + owner_id: str, + *, + task: dict[str, object] | None = None, + ) -> bool: + """Close one settled delivery on a Studio app-server turn. + + Cheap enough for a watcher tick or a read: with the task payload in hand it only + looks at the delivery phase's own bookkeeping, and the turn itself runs on a + background worker because a Codex turn must never sit inside a request. + """ + if not delivery_app_server_enabled(): + return False + try: + session = self._session(task_id, owner_id) + target = self._delivery_turn_target(session, task) + if target is None or not self._delivery_turn_needed(session): + return False + return self._start_app_server_delivery(session, target=target) + except Exception: # noqa: BLE001 - closing a delivery never fails a read + logger.exception( + "Studio migration delivery turn could not start task_id=%s", + task_id, + ) + return False + + def _delivery_turn_target( + self, + session: MigrationSandboxSession, + task: dict[str, object] | None, + ) -> str | None: + """The settled delivery state a closing turn has to explain, if any. + + A closing turn explains how a delivery ended, so it starts on a settled + delivery only: an unfinished run has nothing to report yet, and a structured + migration has no agent work whose outcome needs reading. + """ + if isinstance(task, dict): + state = str(task.get("state") or "") + confirmation = task.get("confirmation") + if ( + state in _DELIVERY_SETTLED_STATES + and isinstance(confirmation, dict) + and confirmation.get("execution_model") == "agentic" + ): + return state + return None + confirmation = self._read_json(session, _CONFIRMATION_PATH, optional=True) + if ( + not isinstance(confirmation, dict) + or confirmation.get("execution_model") != "agentic" + ): + return None + delivery = self._read_json(session, _DELIVERY_STATUS_PATH, optional=True) + if isinstance(delivery, dict): + state = str(delivery.get("state") or "") + if state in _DELIVERY_SETTLED_STATES: + return state + driver = self._read_migration_driver(session) + if self._migration_driver_lost(driver): + return "failed" + process_exit = self._read_json(session, _PROCESS_EXIT_PATH, optional=True) + if process_exit is None: + return None + try: + settled = not self._process_exit_is_settling( + self._validated_process_exit(process_exit) + ) + except MigrationError: + return "failed" + return "failed" if settled else None + + def _delivery_turn_needed(self, session: MigrationSandboxSession) -> bool: + """Whether this delivery still waits for its closing turn. + + The report makes the turn idempotent, and the lease keeps two Studio processes + from closing the same delivery at once: only a lease whose heartbeat stopped + is treated as gone. + """ + running = self._delivery_turns.get(session.session_id) + if running is not None and running.is_alive(): + return False + report = self._read_delivery_report(session) + if isinstance(report, dict): + return False + lease = self._read_delivery_turn_lease(session) + if not isinstance(lease, dict): + return True + if lease.get("state") == _DELIVERY_TURN_RUNNING: + heartbeat = lease.get("heartbeat_at") + age = ( + time.time() - float(heartbeat) + if isinstance(heartbeat, (int, float)) + and not isinstance(heartbeat, bool) + else _DELIVERY_TURN_STALE_SECONDS + ) + if age < _DELIVERY_TURN_STALE_SECONDS: + return False + attempts = lease.get("attempts") + if ( + isinstance(attempts, int) + and not isinstance(attempts, bool) + and attempts >= _DELIVERY_TURN_MAX_ATTEMPTS + and lease.get("verdict") is not True + ): + return False + return True + + def _read_delivery_turn_lease( + self, + session: MigrationSandboxSession, + ) -> dict[str, object] | None: + """Read the closing turn's lease, tolerating a missing or damaged record.""" + try: + lease = self._read_json(session, _DELIVERY_TURN_PATH, optional=True) + except MigrationError: + logger.warning( + "Studio delivery turn lease is unreadable task_id=%s", + session.task_id, + ) + return None + if ( + not isinstance(lease, dict) + or lease.get("schema_version") != 1 + or lease.get("driver") != _DELIVERY_TURN_DRIVER + ): + return None + return lease + + def _read_delivery_report( + self, + session: MigrationSandboxSession, + ) -> dict[str, object] | None: + """Read the closing turn's verdict, tolerating a damaged record. + + The report is an explanation layer on top of the delivery contract, so a record + that cannot be read or validated is ignored rather than failing every later read + of the task. + """ + try: + report = self._read_json(session, _DELIVERY_REPORT_PATH, optional=True) + except MigrationError: + logger.warning( + "Studio delivery report is unreadable task_id=%s", + session.task_id, + ) + return None + if not isinstance(report, dict): + return None + state = str(report.get("state") or "") + if state not in _DELIVERY_SETTLED_STATES: + return None + try: + return validate_delivery_report( + report, + expected_run_id=session.task_id, + expected_state=state, + ) + except MigrationContractError as error: + logger.warning( + "Studio delivery report is invalid task_id=%s error=%s", + session.task_id, + error, + ) + return None + + def _start_app_server_delivery( + self, + session: MigrationSandboxSession, + *, + target: str, + ) -> bool: + """Hand one settled delivery to a Studio background worker.""" + previous = self._read_delivery_turn_lease(session) + attempts = 1 + if isinstance(previous, dict) and isinstance(previous.get("attempts"), int): + attempts = int(previous["attempts"]) + 1 + self._put( + session, + _DELIVERY_TURN_PATH, + _json_bytes( + _delivery_turn_marker(state=_DELIVERY_TURN_RUNNING, attempts=attempts) + ), + media_type="application/json", + ) + worker = threading.Thread( + target=self._app_server_delivery_worker, + args=(session, target, attempts), + name=f"migration-delivery-{session.session_id[-8:]}", + daemon=True, + ) + self._delivery_turns[session.session_id] = worker + try: + worker.start() + except Exception: # noqa: BLE001 - a failed start keeps the CLI's own state + self._delivery_turns.pop(session.session_id, None) + logger.exception( + "Studio migration delivery turn worker could not start task_id=%s", + session.task_id, + ) + return False + return True + + def _app_server_delivery_worker( + self, + session: MigrationSandboxSession, + target: str, + attempts: int, + ) -> None: + """Close one delivery on an app-server turn, or keep the CLI's own record.""" + verdict = False + try: + try: + report = asyncio.run( + self._run_app_server_delivery_turn(session, target=target) + ) + except DeliveryTurnUnavailable as error: + logger.warning( + "Studio migration delivery turn unavailable task_id=%s " + "expected_state=%s error_type=%s", + session.task_id, + target, + type(error).__name__, + ) + report = None + if report is None: + # 没有结论就保留 CLI 自己的交付状态；租约记下这一次没有结论， + # 免得之后每次读任务都重开一个回合。 + logger.warning( + "Studio migration delivery turn returned no verdict; keeping the " + "CLI delivery state task_id=%s expected_state=%s attempts=%s", + session.task_id, + target, + attempts, + ) + else: + verdict = self._persist_delivery_report( + session, + report, + expected_state=target, + ) + except Exception: # noqa: BLE001 - the worker must never kill the process + logger.exception( + "Studio migration delivery turn failed task_id=%s expected_state=%s", + session.task_id, + target, + ) + finally: + if ( + self._delivery_turns.get(session.session_id) + is threading.current_thread() + ): + self._delivery_turns.pop(session.session_id, None) + with contextlib.suppress(Exception): + self._put( + session, + _DELIVERY_TURN_PATH, + _json_bytes( + _delivery_turn_marker( + state=_DELIVERY_TURN_DONE, + attempts=attempts, + verdict=verdict, + ) + ), + media_type="application/json", + ) + + async def _run_app_server_delivery_turn( + self, + session: MigrationSandboxSession, + *, + target: str, + ) -> dict[str, object] | None: + """Run the closing turn while refreshing its lease and recording its events.""" + + async def beat() -> None: + warned = False + while True: + await asyncio.sleep(_DELIVERY_TURN_HEARTBEAT_SECONDS) + try: + await asyncio.to_thread( + self._put, + session, + _DELIVERY_TURN_PATH, + _json_bytes( + _delivery_turn_marker(state=_DELIVERY_TURN_RUNNING) + ), + media_type="application/json", + ) + except Exception as error: # noqa: BLE001 - lease refresh is advisory + if not warned: + warned = True + logger.warning( + "Studio migration delivery turn lease refresh failed " + "task_id=%s error_type=%s", + session.task_id, + type(error).__name__, + ) + + async def questioner( + questions: tuple[dict[str, object], ...], + ) -> dict[str, tuple[str, ...]] | None: + """Publish one question set and wait for the page to answer it.""" + return await self._ask_user( + session, + questions=questions, + attempt=1, + window_seconds=_DELIVERY_TURN_INPUT_WINDOW_SECONDS, + waited_seconds=waited_seconds, + ) + + request = None + try: + request = self._read_json(session, _REQUEST_PATH, optional=True) + except MigrationError: + request = None + model_id = str((request or {}).get("model_id") or "") + framework = "" + exit_code: object = None + try: + confirmation = self._read_json(session, _CONFIRMATION_PATH, optional=True) + if isinstance(confirmation, dict): + framework = str(confirmation.get("framework") or "") + process_exit = self._read_json(session, _PROCESS_EXIT_PATH, optional=True) + if isinstance(process_exit, dict): + exit_code = process_exit.get("exit_code") + except MigrationError: + logger.warning( + "Studio migration delivery turn evidence is incomplete task_id=%s", + session.task_id, + ) + await asyncio.to_thread(self._prepare_delivery_turn_cwd, session) + activity = AnalysisActivityLog( + lambda content: self._put( + session, + _DELIVERY_TURN_ACTIVITY_PATH, + content, + media_type="text/plain", + ) + ) + waited_seconds = [0.0] + heartbeat = asyncio.create_task(beat()) + flusher = asyncio.create_task(activity.run()) + try: + return await run_delivery_turn( + endpoint=session.endpoint, + prompt=_delivery_prompt( + task_id=session.task_id, + framework=framework, + expected_state=target, + exit_code=exit_code, + ), + cwd=_DELIVERY_TURN_CWD, + run_id=session.task_id, + expected_state=target, + publisher=lambda path: self._publish_delivery_artifact( + session, + path, + expected_state=target, + ), + model=model_id, + timeout_seconds=_DELIVERY_TURN_TIMEOUT_SECONDS, + event_sink=activity.record, + extra_tools=( + DynamicTool( + name=ASK_TOOL_NAME, + description=DELIVERY_ASK_TOOL_DESCRIPTION, + schema=ASK_TOOL_SCHEMA, + handler=ask_tool_handler(questioner), + ), + ), + idle_timeout_seconds=( + _DELIVERY_TURN_TIMEOUT_SECONDS + + _DELIVERY_TURN_INPUT_WINDOW_SECONDS + + _DELIVERY_TURN_INPUT_IDLE_MARGIN_SECONDS + ), + host_wait_seconds=lambda: waited_seconds[0], + ) + finally: + flusher.cancel() + with contextlib.suppress(asyncio.CancelledError): + await flusher + await activity.aclose() + heartbeat.cancel() + with contextlib.suppress(asyncio.CancelledError): + await heartbeat + + def _prepare_delivery_turn_cwd(self, session: MigrationSandboxSession) -> None: + """Make sure the turn has a working directory that is not the deliverable.""" + try: + self._execute( + session, + f"mkdir -p {shlex.quote(_DELIVERY_TURN_CWD)}", + operation="prepare_delivery_turn", + timeout_seconds=30, + ) + except Exception as error: # noqa: BLE001 - a cwd is a convenience, not a gate + logger.warning( + "Studio migration delivery turn cwd unavailable task_id=%s " + "error_type=%s", + session.task_id, + type(error).__name__, + ) + + def _publish_delivery_artifact( + self, + session: MigrationSandboxSession, + path: str, + *, + expected_state: str, + ) -> PublishedArtifact: + """Read the delivered artifact back and require it to match the CLI manifest. + + This is why the delivery closes on a Studio turn at all: Studio hashes the bytes + it pulled itself, so a manifest describing some other archive than the one on + disk cannot become this migration's published artifact. + """ + if path != ARTIFACT_PATH: + raise DeliveryContractError(f"产物路径必须是 {ARTIFACT_PATH}") + content = self._read( + session, + _DELIVERY_ARTIFACT_PATH, + max_bytes=_MAX_ARTIFACT_BYTES, + ) + assert content is not None + digest = hashlib.sha256(content).hexdigest() + size = len(content) + manifest = self._read_json(session, _DELIVERY_RESULT_PATH, optional=True) + if not isinstance(manifest, dict): + raise DeliveryContractError("迁移产物清单不存在，无法核对产物") + try: + validated = validate_delivery_result( + manifest, + expected_run_id=session.task_id, + expected_status=expected_state, + ) + except MigrationContractError as error: + raise DeliveryContractError(f"迁移产物清单无效（{error}）") from error + artifact = validated["artifact"] + assert isinstance(artifact, dict) + if artifact.get("sha256") != digest or artifact.get("size") != size: + raise DeliveryContractError("产物字节与迁移产物清单不一致") + return PublishedArtifact(path=ARTIFACT_PATH, sha256=digest, size=size) + + def _persist_delivery_report( + self, + session: MigrationSandboxSession, + report: dict[str, object], + *, + expected_state: str, + ) -> bool: + """Keep the closing turn's verdict beside the delivery it explains.""" + try: + validated = validate_delivery_report( + report, + expected_run_id=session.task_id, + expected_state=expected_state, + ) + except MigrationContractError as error: + logger.warning( + "Studio migration delivery report rejected task_id=%s error=%s", + session.task_id, + error, + ) + return False + self._put( + session, + _DELIVERY_REPORT_PATH, + _json_bytes(validated), + media_type="application/json", + ) + warnings = validated.get("warnings") + logger.info( + "Studio migration delivery turn completed task_id=%s state=%s warnings=%s", + session.task_id, + expected_state, + len(warnings) if isinstance(warnings, list) else 0, + ) + return True + + def _with_delivery_report( + self, + session: MigrationSandboxSession, + task: dict[str, object], + ) -> dict[str, object]: + """Let the closing turn's verdict stand in for the generic CLI sentence. + + The state still comes from the delivery contract; the turn only supplies the + sentence the user reads, and only when it explains that same state. + """ + state = str(task.get("state") or "") + if state not in _DELIVERY_SETTLED_STATES: + return task + report = self._read_delivery_report(session) + if not isinstance(report, dict) or report.get("state") != state: + return task + return {**task, "message": str(report["message"])} + def list_tasks(self, owner_id: str) -> dict[str, list[dict[str, object]]]: try: sessions = self._gateway.list_sessions(owner_id) @@ -3377,7 +4298,8 @@ def list_tasks(self, owner_id: str) -> dict[str, list[dict[str, object]]]: return {"items": tasks} def get_task(self, task_id: str, owner_id: str) -> dict[str, object]: - return self._task_from_session(self._session(task_id, owner_id)) + session = self._session(task_id, owner_id) + return self._with_delivery_report(session, self._task_from_session(session)) @staticmethod def _artifact_status( @@ -3449,6 +4371,17 @@ def _task_payload( "canStop": state in _STOPPABLE_STATES, "artifact": artifact_status, } + # 分析回合和交付收尾回合共用同一张提问卡片：注册表里还有活着的提问， + # 页面就必须看到它。任务状态本身不受影响——分析提问时任务仍是分析中， + # 交付提问时任务已经落定，卡片同样要出现。 + pending_input = self._analysis_input.pending(session.session_id) + if pending_input is not None: + payload["pendingInput"] = ask_payload(pending_input) + payload["message"] = ( + "分析正在等待你的回答" + if state == "analyzing" + else "交付说明正在等待你的回答" + ) if request.get("model_id"): payload["modelId"] = str(request["model_id"]) if isinstance(request.get("evaluation"), dict): @@ -3523,6 +4456,9 @@ def _task_from_session( confirmation, session.task_id, ) + driver = ( + self._read_migration_driver(session) if confirmation is not None else None + ) delivery = self._read_json(session, _DELIVERY_STATUS_PATH, optional=True) delivery_state = "" if delivery is not None: @@ -3601,6 +4537,27 @@ def _task_from_session( "retryable": False, }, ) + if self._migration_driver_lost(driver): + assert driver is not None + logger.warning( + "Studio migration delivery driver stopped without a result " + "task_id=%s heartbeat_at=%s stale_seconds=%s", + session.task_id, + driver.get("heartbeat_at"), + _MIGRATION_DRIVER_STALE_SECONDS, + ) + return self._task_payload( + session, + request, + state="failed", + message="迁移执行进程已中断，请重新发起迁移。", + confirmation=confirmation, + error={ + "code": "MIGRATION_DELIVERY_INTERRUPTED", + "message": "迁移执行进程已中断，未生成完整的迁移交付。", + "retryable": False, + }, + ) if delivery is not None: return self._task_payload( session, @@ -3943,6 +4900,7 @@ def submit_answers( input_sha256=str(source["sha256"]), previous_analysis=analysis, answers=body.answers, + interactive=True, ), attempt=next_attempt, input_sha256=str(source["sha256"]), @@ -3952,6 +4910,65 @@ def submit_answers( self._start_scripted_analysis(session, task_id=task_id, attempt=next_attempt) return self.get_task(task_id, owner_id) + def submit_analysis_input( + self, + task_id: str, + owner_id: str, + body: SubmitAnalysisInputBody, + ) -> dict[str, object]: + """Hand the answers for an in-turn question back to the waiting analysis. + + This is deliberately not the ``needs_input`` re-run: the questions came from the + running app-server turn, so the answers unblock that same turn, which keeps the + project exploration it already paid for. + """ + session = self._session(task_id, owner_id) + pending = self._analysis_input.pending(session.session_id) + if pending is None or pending.request_id != body.request_id: + raise MigrationError( + "MIGRATION_ANALYSIS_INPUT_GONE", + "这次提问已经结束，请刷新页面后按当前分析状态继续。", + status_code=409, + ) + question_ids = [str(question["id"]) for question in pending.questions] + if set(body.answers) - set(question_ids): + raise MigrationError( + "MIGRATION_ANALYSIS_INPUT_INVALID", + "回答与当前分析问题不匹配，请刷新后重试。", + status_code=409, + ) + if any( + not body.answers.get(question_id, "").strip() + for question_id in question_ids + ): + raise MigrationError( + "MIGRATION_ANALYSIS_INPUT_REQUIRED", + "请先回答当前分析的全部问题。", + status_code=422, + ) + try: + answers = normalize_answers( + {question_id: body.answers[question_id] for question_id in question_ids} + ) + except AnalysisAskError as error: + # 请求体已经校验过，这里只是兜底：回答不接受就走同一类错误码。 + raise MigrationError( + "MIGRATION_ANALYSIS_INPUT_INVALID", + f"回答无法提交（{error}）。", + status_code=409, + ) from error + if not self._analysis_input.resolve( + session.session_id, + request_id=body.request_id, + answers=answers, + ): + raise MigrationError( + "MIGRATION_ANALYSIS_INPUT_GONE", + "这次提问已经结束，请刷新页面后按当前分析状态继续。", + status_code=409, + ) + return self.get_task(task_id, owner_id) + def confirm( self, task_id: str, @@ -4167,6 +5184,16 @@ def activity(self, task_id: str, owner_id: str) -> dict[str, object]: items.extend( _parse_activity_log(content, attempt, phase="migration") ) + # 交付收尾回合只在 app-server 上存在，它的事件同样写回 codex exec 行格式， + # 这样迁移页在 CLI 收尾之后还能继续看到 Codex 在做什么。 + turn_log = self._read( + session, + _DELIVERY_TURN_ACTIVITY_PATH, + max_bytes=_MAX_ACTIVITY_LOG_BYTES, + optional=True, + ) + if turn_log is not None: + items.extend(_parse_activity_log(turn_log, 1, phase="delivery")) return { "available": True, "complete": task["state"] in _ACTIVITY_COMPLETE_STATES, @@ -4195,7 +5222,7 @@ def activity(self, task_id: str, owner_id: str) -> dict[str, object]: items: list[dict[str, object]] = [] analysis_log = self._read( session, - f"{MIGRATION_ROOT}/diagnostics/analysis/attempt-{analysis_attempt}.log", + _analysis_activity_path(analysis_attempt), max_bytes=_MAX_ACTIVITY_LOG_BYTES, optional=True, ) @@ -4522,8 +5549,36 @@ def _verified_artifact_content( "迁移产物完整性校验失败。", status_code=502, ) + self._verify_driver_attestation(session, descriptor) return content + def _verify_driver_attestation( + self, + session: MigrationSandboxSession, + descriptor: dict[str, object], + ) -> None: + """Require the published driver digest to agree with the delivery manifest. + + The manifest is produced by the CLI, the digest by the launch script that + supervised it. When both are present they must describe the same archive, + otherwise the bytes changed after the run that produced them. + """ + driver = self._read_migration_driver(session) + if not isinstance(driver, dict) or driver.get("state") != "finished": + return + published = driver.get("artifact") + if not isinstance(published, dict): + return + if ( + published.get("sha256") != descriptor["sha256"] + or published.get("size") != descriptor["size"] + ): + raise MigrationError( + "MIGRATION_ARTIFACT_INTEGRITY_FAILED", + "迁移产物与交付发布清单不一致。", + status_code=502, + ) + def materialize_deployment( self, task_id: str, diff --git a/frontend/src/adk/migrations.ts b/frontend/src/adk/migrations.ts index a964b19ef..511af0f5c 100644 --- a/frontend/src/adk/migrations.ts +++ b/frontend/src/adk/migrations.ts @@ -203,6 +203,18 @@ export interface MigrationAnalysis { warnings: string[]; } +export interface MigrationPendingQuestion { + id: string; + header: string; + question: string; + options: Array<{ label: string; description: string }>; +} + +export interface MigrationPendingInput { + id: string; + questions: MigrationPendingQuestion[]; +} + export interface MigrationTask { id: string; state: MigrationTaskState; @@ -225,6 +237,7 @@ export interface MigrationTask { deployReady: boolean; }; analysis?: MigrationAnalysis; + pendingInput?: MigrationPendingInput; analysisRef?: { attempt: number; sha256: string; @@ -780,6 +793,47 @@ function normalizeAnalysis(value: unknown): MigrationAnalysis { }; } +function normalizePendingInput(value: unknown): MigrationPendingInput { + const input = record(value, adkT("migrations.labels.pendingInput")); + if (typeof input.id !== "string" || !input.id.trim()) { + throw new Error(adkT("migrations.invalidPendingInput")); + } + if (!Array.isArray(input.questions) || input.questions.length === 0) { + throw new Error(adkT("migrations.invalidPendingInput")); + } + const questions = input.questions.map((item) => { + const question = record(item, adkT("migrations.labels.pendingQuestion")); + if ( + typeof question.id !== "string" || + !question.id.trim() || + typeof question.header !== "string" || + typeof question.question !== "string" + ) { + throw new Error(adkT("migrations.invalidPendingInput")); + } + const rawOptions = question.options; + const options = Array.isArray(rawOptions) + ? rawOptions.map((option) => { + const entry = record(option, adkT("migrations.labels.pendingOption")); + if ( + typeof entry.label !== "string" || + typeof entry.description !== "string" + ) { + throw new Error(adkT("migrations.invalidPendingInput")); + } + return { label: entry.label, description: entry.description }; + }) + : []; + return { + id: question.id, + header: question.header, + question: question.question, + options, + }; + }); + return { id: input.id, questions }; +} + function normalizeTask(value: unknown): MigrationTask { const task = record(value, adkT("migrations.labels.task")); const artifact = record( @@ -831,6 +885,8 @@ function normalizeTask(value: unknown): MigrationTask { } if (task.analysis !== undefined) normalized.analysis = normalizeAnalysis(task.analysis); + if (task.pendingInput !== undefined) + normalized.pendingInput = normalizePendingInput(task.pendingInput); if (task.analysisRef !== undefined) { const reference = record( task.analysisRef, @@ -1686,6 +1742,188 @@ export async function getMigrationActivity( ); } +/** One frame of the task event stream; every payload is a whole snapshot. */ +export type MigrationTaskEvent = + | { kind: "task"; seq: number; task: MigrationTask } + | { kind: "activity"; seq: number; activity: MigrationActivity } + /** ``code`` and ``message`` are empty on the frame that reports recovery. */ + | { + kind: "error"; + seq: number; + code: string; + message: string; + retryable: boolean; + } + | { kind: "done"; seq: number; state: string }; + +const STREAM_RETRY_MS = 1_000; +const STREAM_RETRY_MAX_MS = 15_000; + +function invalidStream(): Error { + return new Error(adkT("migrations.invalidEventsResponse")); +} + +/** + * Parse one Server-Sent Events frame into a task event. + * + * Heartbeats, comments, and event names this build does not know yet return ``null`` + * rather than throwing, so a newer Studio can add events without breaking the page. + */ +export function parseMigrationStreamFrame(frame: string): MigrationTaskEvent | null { + const lines = frame.split(/\r?\n/); + const name = lines + .find((line) => line.startsWith("event:")) + ?.slice(6) + .trim(); + if (!name) return null; + const data = lines + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trimStart()) + .join("\n"); + if (!data) return null; + let payload: unknown; + try { + payload = JSON.parse(data); + } catch { + throw invalidStream(); + } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + throw invalidStream(); + } + const value = payload as Record; + const seq = value.seq; + if (typeof seq !== "number" || !Number.isSafeInteger(seq) || seq < 1) { + throw invalidStream(); + } + if (name === "task") return { kind: "task", seq, task: normalizeTask(value) }; + if (name === "activity") { + return { kind: "activity", seq, activity: normalizeActivity(value) }; + } + if (name === "error") { + return { + kind: "error", + seq, + // An error frame without a code or a message says the task is readable again. + code: typeof value.code === "string" ? value.code : "", + message: typeof value.message === "string" ? value.message : "", + retryable: value.retryable === true, + }; + } + if (name === "done") { + return { + kind: "done", + seq, + state: typeof value.state === "string" ? value.state : "", + }; + } + return null; +} + +function streamPause(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve) => { + const finish = () => { + clearTimeout(timer); + signal.removeEventListener("abort", finish); + resolve(); + }; + const timer = setTimeout(finish, ms); + signal.addEventListener("abort", finish, { once: true }); + if (signal.aborted) finish(); + }); +} + +async function readMigrationStream(args: { + taskId: string; + cursor: number; + signal: AbortSignal; + onEvent: (event: MigrationTaskEvent) => void; + advance: (seq: number) => void; +}): Promise { + const response = await request( + `/tasks/${encodeURIComponent(args.taskId)}/events?after=${args.cursor}`, + { + signal: args.signal, + cache: "no-store", + headers: { Accept: "text/event-stream" }, + }, + 0, + ); + if (!response.ok) { + throw await errorFrom(response, adkT("migrations.loadEventsFailed")); + } + if (!response.body) throw invalidStream(); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + let done = false; + try { + while (!done) { + const chunk = await reader.read(); + buffer += decoder.decode(chunk.value, { stream: !chunk.done }); + const frames = buffer.split(/\r?\n\r?\n/); + buffer = frames.pop() ?? ""; + for (const frame of frames) { + const event = parseMigrationStreamFrame(frame); + if (!event) continue; + args.advance(event.seq); + args.onEvent(event); + if (event.kind === "done") done = true; + } + if (chunk.done) break; + } + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + return done; +} + +/** + * Follow one migration task until it settles, reconnecting from the last sequence it + * saw. The server owns the payloads and the settlement rule, so the page no longer + * polls: it applies snapshots and stops when the stream says ``done``. + */ +export async function observeMigrationTask(args: { + taskId: string; + signal: AbortSignal; + after?: number; + onEvent: (event: MigrationTaskEvent) => void; + onConnection?: (message: string) => void; +}): Promise { + let cursor = Math.max(0, Math.trunc(args.after ?? 0)); + let retry = 0; + while (!args.signal.aborted) { + try { + const done = await readMigrationStream({ + taskId: args.taskId, + cursor, + signal: args.signal, + onEvent: args.onEvent, + advance: (seq) => { + cursor = seq; + }, + }); + retry = 0; + if (done || args.signal.aborted) return; + args.onConnection?.(""); + await streamPause(STREAM_RETRY_MS, args.signal); + } catch (error) { + if (args.signal.aborted) return; + if ( + error instanceof MigrationApiError && + [401, 403, 404].includes(error.status) + ) { + throw error; + } + args.onConnection?.(adkT("migrations.reconnecting")); + await streamPause( + Math.min(STREAM_RETRY_MS * 2 ** retry++, STREAM_RETRY_MAX_MS), + args.signal, + ); + } + } +} + export async function confirmMigrationTask(args: { taskId: string; framework: MigrationFramework; @@ -1753,6 +1991,32 @@ export async function submitMigrationAnalysisAnswers(args: { ); } +export async function submitMigrationAnalysisInput(args: { + taskId: string; + requestId: string; + answers: Record; + signal?: AbortSignal; +}): Promise { + return normalizeTask( + await json( + await request( + `/tasks/${encodeURIComponent(args.taskId)}/input`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + requestId: args.requestId, + answers: args.answers, + }), + signal: args.signal, + }, + SESSION_START_TIMEOUT_MS, + ), + adkT("migrations.submitInputFailed"), + ), + ); +} + export async function stopMigrationTask( taskId: string, signal?: AbortSignal, diff --git a/frontend/src/i18n/resources/en-US/adk.json b/frontend/src/i18n/resources/en-US/adk.json index 25d3e7df5..c47fc09e6 100644 --- a/frontend/src/i18n/resources/en-US/adk.json +++ b/frontend/src/i18n/resources/en-US/adk.json @@ -170,6 +170,7 @@ "invalidAnalysisEvidence": "The analysis evidence has an invalid format.", "invalidEntryCandidate": "An entry candidate has an invalid format.", "invalidQuestion": "A follow-up question has an invalid format.", + "invalidPendingInput": "Analysis questions are not readable", "invalidTask": "The migration session has an invalid format.", "invalidAnalysisReference": "The analysis result reference has an invalid format.", "invalidSourcePersistence": "The migration source persistence status has an invalid format.", @@ -193,8 +194,12 @@ "createTaskFailed": "Failed to create the migration session", "uploadProjectFailed": "Failed to upload the migration project", "loadActivityFailed": "Failed to load migration activity", + "loadEventsFailed": "Failed to follow the migration progress stream", + "invalidEventsResponse": "The migration progress stream returned an invalid frame.", + "reconnecting": "Connection lost. Reconnecting to the migration progress stream…", "startFailed": "Failed to start the migration", "submitAnswersFailed": "Failed to submit additional analysis information", + "submitInputFailed": "Could not submit the analysis answer", "stopFailed": "Failed to stop the migration", "deleteTaskFailed": "Failed to delete the migration session", "loadArtifactFailed": "Failed to load the migration artifact", @@ -268,7 +273,10 @@ "evaluationSummary": "Evaluation summary", "evaluationCaseResult": "Evaluation case result", "evaluationOutput": "Evaluation output", - "evaluationLimitations": "Evaluation limitations" + "evaluationLimitations": "Evaluation limitations", + "pendingInput": "Analysis question", + "pendingQuestion": "Analysis question item", + "pendingOption": "Analysis question option" } }, "sandbox": { diff --git a/frontend/src/i18n/resources/en-US/migrations.json b/frontend/src/i18n/resources/en-US/migrations.json index 2c86b0279..b5ef94b40 100644 --- a/frontend/src/i18n/resources/en-US/migrations.json +++ b/frontend/src/i18n/resources/en-US/migrations.json @@ -213,6 +213,15 @@ "submitting": "Continuing analysis…", "submit": "Submit and continue analysis" }, + "pendingInput": { + "ariaLabel": "Answer the questions the migration needs you to decide", + "title": "The migration needs your answer", + "description": "Your answer continues the current step instead of restarting it", + "other": "Other", + "otherPlaceholder": "Or type your own answer", + "submitting": "Submitting your answer…", + "submit": "Submit answer and continue" + }, "confirmation": { "ariaLabel": "Confirm migration method", "title": "Confirm migration method", diff --git a/frontend/src/i18n/resources/zh-CN/adk.json b/frontend/src/i18n/resources/zh-CN/adk.json index f7b8a6487..ea385df8e 100644 --- a/frontend/src/i18n/resources/zh-CN/adk.json +++ b/frontend/src/i18n/resources/zh-CN/adk.json @@ -170,6 +170,7 @@ "invalidAnalysisEvidence": "分析证据格式错误。", "invalidEntryCandidate": "入口候选格式错误。", "invalidQuestion": "待确认问题格式错误。", + "invalidPendingInput": "分析问题无法读取", "invalidTask": "迁移会话格式错误。", "invalidAnalysisReference": "分析结果引用格式错误。", "invalidSourcePersistence": "迁移源码保存状态格式错误。", @@ -193,8 +194,12 @@ "createTaskFailed": "创建迁移会话失败", "uploadProjectFailed": "上传迁移项目失败", "loadActivityFailed": "读取迁移执行动态失败", + "loadEventsFailed": "订阅迁移进度流失败", + "invalidEventsResponse": "迁移进度流返回了非法数据帧。", + "reconnecting": "连接已断开，正在重新连接迁移进度流…", "startFailed": "启动迁移失败", "submitAnswersFailed": "提交分析补充信息失败", + "submitInputFailed": "提交分析回答失败", "stopFailed": "终止迁移失败", "deleteTaskFailed": "删除迁移会话失败", "loadArtifactFailed": "读取迁移产物失败", @@ -268,7 +273,10 @@ "evaluationSummary": "评测汇总", "evaluationCaseResult": "评测用例结果", "evaluationOutput": "评测输出", - "evaluationLimitations": "评测限制" + "evaluationLimitations": "评测限制", + "pendingInput": "分析提问", + "pendingQuestion": "分析问题项", + "pendingOption": "分析问题选项" } }, "sandbox": { diff --git a/frontend/src/i18n/resources/zh-CN/migrations.json b/frontend/src/i18n/resources/zh-CN/migrations.json index 15e94a47d..a590a41b7 100644 --- a/frontend/src/i18n/resources/zh-CN/migrations.json +++ b/frontend/src/i18n/resources/zh-CN/migrations.json @@ -213,6 +213,15 @@ "submitting": "正在继续分析…", "submit": "提交并继续分析" }, + "pendingInput": { + "ariaLabel": "回答迁移需要你决定的问题", + "title": "迁移需要你的回答", + "description": "回答后会在当前这一步里继续，不需要重新开始", + "other": "其他", + "otherPlaceholder": "也可以直接输入你的答案", + "submitting": "正在提交回答…", + "submit": "提交回答并继续" + }, "confirmation": { "ariaLabel": "确认迁移方式", "title": "确认迁移方式", diff --git a/frontend/src/migrations/MigrationWorkspace.css b/frontend/src/migrations/MigrationWorkspace.css index 5321c97a4..520432f3e 100644 --- a/frontend/src/migrations/MigrationWorkspace.css +++ b/frontend/src/migrations/MigrationWorkspace.css @@ -857,6 +857,113 @@ font-weight: 400; } +/* Questions asked inside the running analysis turn: options plus a free-text answer. */ +.migration-question { + min-width: 0; + display: grid; + gap: 8px; + overflow-wrap: anywhere; +} + +.migration-question__header { + color: hsl(var(--foreground)); + font-size: 12.5px; + font-weight: 600; +} + +.migration-question__prompt { + margin: 0; + color: hsl(var(--foreground)); + font-size: 13px; + font-weight: 400; + line-height: 1.55; +} + +.migration-question__options { + display: grid; + gap: 6px; +} + +.migration-question__option { + display: flex; + gap: 9px; + align-items: flex-start; + padding: 9px 11px; + border: 1px solid hsl(var(--border)); + border-radius: 8px; + cursor: pointer; + transition: border-color 120ms ease, background 120ms ease; +} + +.migration-question__option:hover { + border-color: hsl(var(--ring) / 0.45); +} + +.migration-question__option.is-selected { + border-color: hsl(var(--ring) / 0.7); + background: hsl(var(--accent) / 0.5); +} + +.migration-question__option input { + flex: 0 0 auto; + width: 15px; + height: 15px; + margin: 2px 0 0; + accent-color: hsl(var(--primary)); +} + +.migration-question__option span { + min-width: 0; + display: grid; + gap: 2px; +} + +.migration-question__option strong { + color: hsl(var(--foreground)); + font-size: 12.5px; + font-weight: 600; +} + +.migration-question__option em { + color: hsl(var(--muted-foreground)); + font-size: 12px; + font-style: normal; + line-height: 1.5; +} + +.migration-question__other { + display: grid; + gap: 6px; + color: hsl(var(--muted-foreground)); + font-size: 12px; +} + +.migration-question__other textarea { + width: 100%; + min-height: 40px; + box-sizing: border-box; + padding: 9px 11px; + border: 1px solid hsl(var(--border)); + border-radius: 8px; + outline: 0; + background: hsl(var(--background)); + color: hsl(var(--foreground)); + font: inherit; + font-size: 13.5px; + line-height: 1.55; + resize: vertical; +} + +.migration-question__other textarea:focus { + border-color: hsl(var(--ring) / 0.7); + box-shadow: 0 0 0 2px hsl(var(--ring) / 0.12); +} + +.migration-question__other textarea:disabled { + cursor: not-allowed; + opacity: 0.6; +} + .migration-confirmation__actions { display: flex; justify-content: flex-end; diff --git a/frontend/src/migrations/MigrationWorkspace.tsx b/frontend/src/migrations/MigrationWorkspace.tsx index b9414a1c2..cb942488c 100644 --- a/frontend/src/migrations/MigrationWorkspace.tsx +++ b/frontend/src/migrations/MigrationWorkspace.tsx @@ -17,7 +17,6 @@ import { createMigrationTask, downloadMigrationArtifact, downloadMigrationEvaluationReport, - getMigrationActivity, getMigrationArtifact, getMigrationArtifactFile, getMigrationCapabilities, @@ -25,12 +24,14 @@ import { getMigrationEvaluationReport, getMigrationTask, listMigrationTasks, + observeMigrationTask, MigrationApiError, putMigrationEvaluationDataset, resumeMigrationEvaluation, retryMigrationEvaluation, stopMigrationTask, submitMigrationAnalysisAnswers, + submitMigrationAnalysisInput, uploadMigrationSource, type MigrationAnalysis, type MigrationActivity, @@ -105,8 +106,8 @@ import { i18n } from "../i18n/runtime"; import "./MigrationWorkspace.css"; const MAX_SOURCE_BYTES = 20 * 1024 * 1024; -const POLL_INTERVAL_MS = 1_200; -const ACTIVITY_POLL_INTERVAL_MS = 3_000; +// Task detail and activity arrive on one Server-Sent Events stream; only the +// list of recent tasks is still polled. const LIST_POLL_INTERVAL_MS = 5_000; const MAX_VISIBLE_FILES = 500; const ignoreMigrationAction = () => undefined; @@ -732,6 +733,7 @@ export function MigrationWorkspace({ const evaluationTabRef = useRef(null); const preparedAnalysisRef = useRef(""); const evaluationDraftTaskRef = useRef(""); + const taskEventCursorRef = useRef({ taskId: "", seq: 0 }); const transferAbortRef = useRef(null); const evaluationReportAbortRef = useRef(null); const [capability, setCapability] = useState( @@ -757,7 +759,14 @@ export function MigrationWorkspace({ const [loadKey, setLoadKey] = useState(0); const [showAllTasks, setShowAllTasks] = useState(false); const [action, setAction] = useState< - "create" | "upload" | "answer" | "confirm" | "stop" | "download" | "" + | "create" + | "upload" + | "answer" + | "input" + | "confirm" + | "stop" + | "download" + | "" >(""); const [error, setError] = useState(""); const [pollError, setPollError] = useState(""); @@ -768,6 +777,10 @@ export function MigrationWorkspace({ const [entry, setEntry] = useState(""); const [appName, setAppName] = useState(""); const [answers, setAnswers] = useState>({}); + // Answers for questions asked inside the running analysis turn. They are kept + // apart from the needs_input answers: the two cards are mutually exclusive and + // carry different payloads. + const [inputAnswers, setInputAnswers] = useState>({}); const [artifact, setArtifact] = useState(null); const [artifactError, setArtifactError] = useState(""); const [artifactErrorRetryable, setArtifactErrorRetryable] = useState(false); @@ -1072,55 +1085,74 @@ export function MigrationWorkspace({ }, [action, hasPollableTasks]); useEffect(() => { + if (action === "confirm" || !task || taskEnvironmentExpired) return; if ( - action === "confirm" || - !task || - taskEnvironmentExpired || - (!isActiveState(task.state) && - task.persistence?.state !== "saving" && - !isEvaluationPollingState(task)) + !shouldShowCodexActivity(task) && + !isActiveState(task.state) && + task.persistence?.state !== "saving" && + !isEvaluationPollingState(task) ) return; const controller = new AbortController(); - let timer: number | undefined; - const poll = async () => { - try { - const next = await getMigrationTask(task.id, controller.signal); + // Task detail and activity are one snapshot stream now, so the page stops polling + // both. The effect restarts whenever the server-side settlement rule can change; + // the cursor keeps those restarts cheap by resuming instead of replaying. + const resume = + taskEventCursorRef.current.taskId === task.id + ? taskEventCursorRef.current.seq + : 0; + setActivityLoading(activity === null && shouldShowCodexActivity(task)); + void observeMigrationTask({ + taskId: task.id, + signal: controller.signal, + after: resume, + onEvent: (event) => { if (controller.signal.aborted) return; - setTasks((current) => upsertTask(current, next)); + taskEventCursorRef.current = { taskId: task.id, seq: event.seq }; + if (event.kind === "error") { + setActivityLoading(false); + // A frame without code or message says the task is readable again. + const cleared = !event.code && !event.message; + setPollError(cleared ? "" : event.message || t("activity.loadError")); + setPollErrorRetryable(cleared ? false : event.retryable); + return; + } setPollError(""); setPollErrorRetryable(false); - if ( - !isMigrationEnvironmentExpired(next, Date.now()) && - (isActiveState(next.state) || - next.persistence?.state === "saving" || - isEvaluationPollingState(next)) - ) { - timer = window.setTimeout(() => void poll(), POLL_INTERVAL_MS); + if (event.kind === "task") { + setTasks((current) => upsertTask(current, event.task)); + return; } - } catch (cause) { - if (controller.signal.aborted) return; - setPollError(cause instanceof Error ? cause.message : String(cause)); - setPollErrorRetryable( - cause instanceof MigrationApiError && cause.retryable, - ); - if (cause instanceof MigrationApiError && cause.retryable) { - timer = window.setTimeout(() => void poll(), POLL_INTERVAL_MS); + if (event.kind === "activity") { + setActivity(event.activity); + setActivityError(""); + setActivityLoading(false); + return; } - } - }; - timer = window.setTimeout(() => void poll(), POLL_INTERVAL_MS); - return () => { - controller.abort(); - if (timer !== undefined) window.clearTimeout(timer); - }; + setActivityLoading(false); + }, + onConnection: (message) => { + if (controller.signal.aborted) return; + setPollError(message); + setPollErrorRetryable(false); + }, + }).catch((cause: unknown) => { + if (controller.signal.aborted) return; + setActivityLoading(false); + setPollError(cause instanceof Error ? cause.message : String(cause)); + setPollErrorRetryable(cause instanceof MigrationApiError && cause.retryable); + }); + return () => controller.abort(); }, [ + action, task?.id, task?.state, + task?.analysisRef?.sha256, + task?.confirmation?.framework, task?.persistence?.state, task?.evaluation?.state, taskEnvironmentExpired, - action, + t, ]); useEffect(() => { @@ -1141,62 +1173,6 @@ export function MigrationWorkspace({ setActivityLoading(false); }, [task?.id]); - useEffect(() => { - if (!task || taskEnvironmentExpired || !shouldShowCodexActivity(task)) { - return; - } - - const controller = new AbortController(); - let timer: number | undefined; - const poll = async () => { - setActivityLoading(true); - try { - const next = await getMigrationActivity(task.id, controller.signal); - if (controller.signal.aborted) return; - setActivity(next); - setActivityError(""); - if ( - !next.complete && - !taskEnvironmentExpired && - isActiveState(task.state) - ) { - timer = window.setTimeout( - () => void poll(), - ACTIVITY_POLL_INTERVAL_MS, - ); - } - } catch (cause) { - if (controller.signal.aborted) return; - setActivityError(t("activity.loadError")); - if ( - !taskEnvironmentExpired && - isActiveState(task.state) && - cause instanceof MigrationApiError && - cause.retryable - ) { - timer = window.setTimeout( - () => void poll(), - ACTIVITY_POLL_INTERVAL_MS, - ); - } - } finally { - if (!controller.signal.aborted) setActivityLoading(false); - } - }; - void poll(); - return () => { - controller.abort(); - if (timer !== undefined) window.clearTimeout(timer); - }; - }, [ - task?.id, - task?.state, - task?.analysisRef?.sha256, - task?.confirmation?.framework, - taskEnvironmentExpired, - t, - ]); - useEffect(() => { if ( !task?.analysis || @@ -1614,6 +1590,44 @@ export function MigrationWorkspace({ requiredQuestionsAnswered, ); + // The card follows a live question, not the state: a delivery turn asks after the delivery has already settled. + const pendingInput = task?.pendingInput; + const pendingInputAnswered = (pendingInput?.questions ?? []).every( + (question) => (inputAnswers[question.id] || "").trim().length > 0, + ); + const canSubmitInput = Boolean(pendingInput && pendingInputAnswered && !action); + + // Every question set is a new one, so the previous draft must not leak into it. + const pendingInputId = pendingInput?.id; + useEffect(() => { + setInputAnswers({}); + }, [pendingInputId]); + + async function submitInput() { + if (!task || !pendingInput || !canSubmitInput) return; + setAction("input"); + setError(""); + try { + const next = await submitMigrationAnalysisInput({ + taskId: task.id, + requestId: pendingInput.id, + answers: Object.fromEntries( + pendingInput.questions.map((question) => [ + question.id, + (inputAnswers[question.id] || "").trim(), + ]), + ), + }); + setTasks((current) => upsertTask(current, next)); + } catch (cause) { + const authoritative = await reconcileTaskState(task.id); + if (authoritative && !authoritative.pendingInput) return; + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setAction(""); + } + } + async function submitAnswers() { if (!task?.analysisRef || !canSubmitAnswers) return; setAction("answer"); @@ -1972,7 +1986,7 @@ export function MigrationWorkspace({ const composerFile = sourceFile; const composerBusy = action === "create" || action === "upload"; const isHome = page === "new" && !task && action !== "create"; - const navigationBusy = composerBusy || action === "confirm" || action === "answer" || action === "stop" || Boolean(evaluationAction); + const navigationBusy = composerBusy || action === "confirm" || action === "answer" || action === "input" || action === "stop" || Boolean(evaluationAction); const showComposer = !task || (task.canUpload && !taskEnvironmentExpired); const expiryCopy = task ? migrationExpiryCopy(task, now) : null; const hasEvaluationTab = Boolean(task?.evaluation?.enabled); @@ -2496,6 +2510,97 @@ export function MigrationWorkspace({ )} + {pendingInput ? ( +
+
+ {t("pendingInput.title")} + {t("pendingInput.description")} +
+ {pendingInput.questions.map((question) => { + const selected = inputAnswers[question.id] || ""; + const chosen = question.options.some( + (option) => option.label === selected, + ); + return ( +
+ + {question.header} + +

+ {question.question} +

+ {question.options.length > 0 ? ( +
+ {question.options.map((option) => ( + + ))} +
+ ) : null} +