diff --git a/README.md b/README.md index ac7f1df..b3891e8 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,30 @@ Run `th-cli test-run-execution --id {id}` with a test run execution id to fetch For JSON respond, add `--json` to the command. +### Test Run Execution Repeat + +Run `th-cli test-run-execution repeat --id {id}` to create a new test run execution with +the same selected tests and config as an existing one, then start it and attach to it the +same way `run-tests` does (and the frontend's "Repeat" action does): streaming live test +progress and forwarding any user prompts to this terminal. Use `--title` to override the +generated title (defaults to the original title with an updated timestamp; the backend +always appends a timestamp regardless). As with `run-tests`, `--no-color` disables colored +output and `--no-streaming` disables the real-time web log viewer (enabled by default) for +the new execution. + +### Test Run Execution Export + +Run `th-cli test-run-execution export --id {id}` to export a test run execution's config +and results to a JSON file. Use `--output-file` to override the default filename +(`-execution.json`). + +### Test Run Execution Import + +Run `th-cli test-run-execution import --file {file} --project-id {id}` to import a test +run execution previously written by `test-run-execution export` into the given project. +The backend rejects the import if the file's `db_revision` doesn't match the destination +instance's current database revision. + ### Project Create Run `th-cli project create --name {project name} --config {config file}` to create a new project. Project name is required. diff --git a/tests/test_client.py b/tests/test_client.py index ced7a92..b336577 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -86,6 +86,33 @@ def capture(**kwargs): assert captured_host[0].startswith("http://") + def test_timeout_not_passed_when_omitted(self): + """When no timeout is given, ApiClient is not passed a 'timeout' kwarg (httpx's + own default applies).""" + from th_cli.client import get_client + + with patch("th_cli.client.ApiClient") as mock_cls: + mock_cls.return_value = MagicMock() + get_client() + + assert "timeout" not in mock_cls.call_args.kwargs + + def test_timeout_forwarded_to_api_client(self): + """An explicit timeout is forwarded to ApiClient (and from there to the + underlying httpx.AsyncClient), instead of callers having to poke + client._async_client.timeout after construction.""" + from httpx import Timeout + + from th_cli.client import get_client + + custom_timeout = Timeout(120.0, connect=10.0) + + with patch("th_cli.client.ApiClient") as mock_cls: + mock_cls.return_value = MagicMock() + get_client(timeout=custom_timeout) + + assert mock_cls.call_args.kwargs.get("timeout") == custom_timeout + # --------------------------------------------------------------------------- # Module-level client fallback diff --git a/tests/test_test_run_execution_repeat_export_import.py b/tests/test_test_run_execution_repeat_export_import.py new file mode 100644 index 0000000..7239f01 --- /dev/null +++ b/tests/test_test_run_execution_repeat_export_import.py @@ -0,0 +1,580 @@ +# +# Copyright (c) 2026 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for the `test-run-execution repeat/export/import` commands.""" + +import json +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from click.testing import CliRunner +from httpx import ReadTimeout + +from th_cli.api_lib_autogen import models as api_models +from th_cli.api_lib_autogen.exceptions import ResponseHandlingException, UnexpectedResponse +from th_cli.commands.test_run_execution import test_run_execution +from th_cli.exceptions import ConfigurationError + + +def _make_exported_execution(title: str = "My Execution!") -> api_models.ExportedTestRunExecution: + """Build a minimal ExportedTestRunExecution for export/import round-trip tests.""" + from datetime import datetime, timezone + + return api_models.ExportedTestRunExecution( + db_revision="abc123", + test_run_execution=api_models.TestRunExecutionToExport( + title=title, + state=api_models.TestStateEnum.passed, + created_at=datetime.now(timezone.utc), + log=[], + ), + ) + + +@pytest.mark.unit +@pytest.mark.cli +class TestRepeatCommand: + """Test cases for the `test-run-execution repeat` command.""" + + @pytest.fixture(autouse=True) + def mock_test_logging(self): + """Patch th_cli.test_run.logging so tests don't start a real LogStreamHandler + (and its daemon HTTP server) for every invocation.""" + with patch("th_cli.commands.test_run_execution.test_logging") as mock_logging: + mock_logging.configure_logger_for_run.return_value = "/tmp/test_run.log" + mock_logging.get_log_stream_url.return_value = None + yield mock_logging + + def test_repeat_success( + self, + cli_runner: CliRunner, + mock_async_apis: Mock, + mock_api_client: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + mock_test_logging: Mock, + ) -> None: + """By default, repeat creates the new execution, starts it, and attaches to it the + same way `run-tests` does (and the frontend's 'Repeat' action does): streaming live + progress instead of just reporting a static message.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.return_value = ( + sample_test_run_execution + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.return_value = ( + sample_test_run_execution + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + patch("th_cli.commands.test_run_execution.TestRunSocket") as mock_socket_class, + ): + mock_socket = Mock() + mock_socket.connect_websocket = AsyncMock() + mock_socket_class.return_value = mock_socket + + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 0 + assert f"repeated as new execution {sample_test_run_execution.id}" in result.output + assert sample_test_run_execution.title in result.output + assert "Starting Test run" in result.output + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.assert_called_once_with( + id=1, title=None + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.assert_called_once_with( + id=sample_test_run_execution.id + ) + mock_test_logging.configure_logger_for_run.assert_called_once_with( + title=sample_test_run_execution.title, enable_log_streaming=True + ) + mock_socket_class.assert_called_once_with( + sample_test_run_execution, project_config_dict=sample_test_run_execution.execution_config or {} + ) + mock_socket.connect_websocket.assert_called_once() + assert mock_socket.run == sample_test_run_execution + mock_test_logging.stop_log_streaming.assert_called_once() + mock_api_client.aclose.assert_called_once() + + def test_repeat_with_custom_title( + self, + cli_runner: CliRunner, + mock_async_apis: Mock, + mock_api_client: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + ) -> None: + """--title is forwarded to the API call.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.return_value = ( + sample_test_run_execution + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.return_value = ( + sample_test_run_execution + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + patch("th_cli.commands.test_run_execution.TestRunSocket") as mock_socket_class, + ): + mock_socket = Mock() + mock_socket.connect_websocket = AsyncMock() + mock_socket_class.return_value = mock_socket + + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1", "--title", "Custom Title"]) + + assert result.exit_code == 0 + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.assert_called_once_with( + id=1, title="Custom Title" + ) + + def test_repeat_start_api_error( + self, + cli_runner: CliRunner, + mock_async_apis: Mock, + mock_api_client: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + mock_test_logging: Mock, + ) -> None: + """A failure to start the repeated execution is surfaced via the standard error-handling path, + and the log streaming server started for the run is still stopped.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.return_value = ( + sample_test_run_execution + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.side_effect = UnexpectedResponse( + status_code=500, content=b"Internal Server Error" + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + patch("th_cli.commands.test_run_execution.TestRunSocket") as mock_socket_class, + ): + mock_socket = Mock() + mock_socket.connect_websocket = AsyncMock() + mock_socket_class.return_value = mock_socket + + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 1 + assert ( + f"Failed to start repeated test run execution '{sample_test_run_execution.id}' " + "(Status: 500) - Internal Server Error" in result.output + ) + mock_test_logging.stop_log_streaming.assert_called_once() + + def test_repeat_start_conflict( + self, + cli_runner: CliRunner, + mock_async_apis: Mock, + mock_api_client: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + mock_test_logging: Mock, + ) -> None: + """A 409 (e.g. test engine busy) makes clear the execution was still created, and the + log streaming server started for the run is still stopped.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.return_value = ( + sample_test_run_execution + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.side_effect = UnexpectedResponse( + status_code=409, content={"detail": "Test Engine is busy."} + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + patch("th_cli.commands.test_run_execution.TestRunSocket") as mock_socket_class, + ): + mock_socket = Mock() + mock_socket.connect_websocket = AsyncMock() + mock_socket_class.return_value = mock_socket + + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 1 + assert f"Execution {sample_test_run_execution.id} was created but could not be started" in result.output + assert "Test Engine is busy." in result.output + mock_test_logging.stop_log_streaming.assert_called_once() + + def test_repeat_start_timeout( + self, + cli_runner: CliRunner, + mock_async_apis: Mock, + mock_api_client: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + mock_test_logging: Mock, + ) -> None: + """A timeout while starting the repeated execution is surfaced as a clean, readable + error instead of a raw traceback, and the log streaming server started for the run + is still stopped.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.return_value = ( + sample_test_run_execution + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.side_effect = ( + ResponseHandlingException(ReadTimeout("timed out")) + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + patch("th_cli.commands.test_run_execution.TestRunSocket") as mock_socket_class, + ): + mock_socket = Mock() + mock_socket.connect_websocket = AsyncMock() + mock_socket_class.return_value = mock_socket + + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 1 + assert "Timed out waiting for the server" in result.output + mock_test_logging.stop_log_streaming.assert_called_once() + + def test_repeat_websocket_connect_failure( + self, + cli_runner: CliRunner, + mock_async_apis: Mock, + mock_api_client: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + mock_test_logging: Mock, + ) -> None: + """An unexpected failure while connecting the websocket (e.g. the backend refuses the + connection) is surfaced as a clean CLIError instead of a raw Python traceback, and the + log streaming server started for the run is still stopped.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.return_value = ( + sample_test_run_execution + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.return_value = ( + sample_test_run_execution + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + patch("th_cli.commands.test_run_execution.TestRunSocket") as mock_socket_class, + ): + mock_socket = Mock() + mock_socket.connect_websocket = AsyncMock(side_effect=ConnectionRefusedError("connection refused")) + mock_socket_class.return_value = mock_socket + + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 1 + assert "Unexpected error during repeated test execution" in result.output + assert "Traceback" not in result.output + mock_test_logging.stop_log_streaming.assert_called_once() + mock_api_client.aclose.assert_called_once() + + def test_repeat_not_found(self, cli_runner: CliRunner, mock_async_apis: Mock, mock_api_client: Mock) -> None: + """A 404 from the API is surfaced as a clear 'not found' error, and nothing is started.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.side_effect = UnexpectedResponse( + status_code=404, content={"detail": "TestRunExecution not found"} + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + ): + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "999"]) + + assert result.exit_code == 1 + assert "Test run execution with ID '999' not found." in result.output + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.assert_not_called() + + def test_repeat_other_api_error( + self, cli_runner: CliRunner, mock_async_apis: Mock, mock_api_client: Mock + ) -> None: + """A non-404 API error is surfaced via the standard error-handling path, and nothing is started.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.side_effect = UnexpectedResponse( + status_code=500, content=b"Internal Server Error" + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + ): + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 1 + assert "Failed to repeat test run execution '1' (Status: 500) - Internal Server Error" in result.output + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.assert_not_called() + + def test_repeat_timeout_error( + self, cli_runner: CliRunner, mock_async_apis: Mock, mock_api_client: Mock + ) -> None: + """A timeout while repeating the execution is surfaced as a clean, readable error + instead of a raw traceback (the original bug report for this command).""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.side_effect = ( + ResponseHandlingException(ReadTimeout("timed out")) + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + ): + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 1 + assert "Timed out waiting for the server" in result.output + + def test_repeat_configuration_error(self, cli_runner: CliRunner) -> None: + """A ConfigurationError from get_client is surfaced to the user.""" + with patch( + "th_cli.commands.test_run_execution.get_client", + side_effect=ConfigurationError("Could not connect to server"), + ): + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1"]) + + assert result.exit_code == 1 + assert "Error: Could not connect to server" in result.output + + def test_repeat_requires_id(self, cli_runner: CliRunner) -> None: + """The --id parameter is required.""" + result = cli_runner.invoke(test_run_execution, ["repeat"]) + + assert result.exit_code != 0 + assert "Missing option" in result.output or "--id" in result.output + + def test_repeat_no_streaming_disables_log_viewer( + self, + cli_runner: CliRunner, + mock_async_apis: Mock, + mock_api_client: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + mock_test_logging: Mock, + ) -> None: + """--no-streaming disables the real-time web log viewer for the new execution.""" + api = mock_async_apis.test_run_executions_api + api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post.return_value = ( + sample_test_run_execution + ) + api.start_test_run_execution_api_v1_test_run_executions__id__start_post.return_value = ( + sample_test_run_execution + ) + + with ( + patch("th_cli.commands.test_run_execution.get_client", return_value=mock_api_client), + patch("th_cli.commands.test_run_execution.AsyncApis", return_value=mock_async_apis), + patch("th_cli.commands.test_run_execution.TestRunSocket") as mock_socket_class, + ): + mock_socket = Mock() + mock_socket.connect_websocket = AsyncMock() + mock_socket_class.return_value = mock_socket + + result = cli_runner.invoke(test_run_execution, ["repeat", "--id", "1", "--no-streaming"]) + + assert result.exit_code == 0 + mock_test_logging.configure_logger_for_run.assert_called_once_with( + title=sample_test_run_execution.title, enable_log_streaming=False + ) + + def test_repeat_help_message(self, cli_runner: CliRunner) -> None: + """Test the help message for the repeat command.""" + result = cli_runner.invoke(test_run_execution, ["repeat", "--help"]) + + assert result.exit_code == 0 + assert "--id" in result.output + assert "--title" in result.output + assert "--no-color" in result.output + assert "--no-streaming" in result.output + + +@pytest.mark.unit +@pytest.mark.cli +class TestExportExecutionCommand: + """Test cases for the `test-run-execution export` command.""" + + def test_export_success_default_filename(self, cli_runner: CliRunner, mock_sync_apis: Mock) -> None: + """When no --output-file is given, the file is named after the execution title.""" + exported = _make_exported_execution(title="My Execution!") + api = mock_sync_apis.test_run_executions_api + api.export_test_run_execution_api_v1_test_run_executions__id__export_get.return_value = exported + + with cli_runner.isolated_filesystem(): + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(test_run_execution, ["export", "--id", "1"]) + + assert result.exit_code == 0 + assert Path("MyExecution-execution.json").exists() + saved = json.loads(Path("MyExecution-execution.json").read_text()) + assert saved["test_run_execution"]["title"] == "My Execution!" + + api.export_test_run_execution_api_v1_test_run_executions__id__export_get.assert_called_once_with(id=1) + + def test_export_success_custom_filename(self, cli_runner: CliRunner, mock_sync_apis: Mock, temp_dir: Path) -> None: + """When --output-file is given, the export is written there.""" + exported = _make_exported_execution() + api = mock_sync_apis.test_run_executions_api + api.export_test_run_execution_api_v1_test_run_executions__id__export_get.return_value = exported + output_path = str(temp_dir / "run-42.json") + + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(test_run_execution, ["export", "--id", "42", "--output-file", output_path]) + + assert result.exit_code == 0 + assert f"exported to '{output_path}'" in result.output + assert Path(output_path).exists() + + def test_export_not_found(self, cli_runner: CliRunner, mock_sync_apis: Mock) -> None: + """A 404 from the API is surfaced as a clear 'not found' error.""" + api = mock_sync_apis.test_run_executions_api + api.export_test_run_execution_api_v1_test_run_executions__id__export_get.side_effect = UnexpectedResponse( + status_code=404, content={"detail": "Test Run Execution with id 999 not found"} + ) + + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(test_run_execution, ["export", "--id", "999"]) + + assert result.exit_code == 1 + assert "Test run execution with ID '999' not found." in result.output + + def test_export_write_failure_raises_cli_error(self, cli_runner: CliRunner, mock_sync_apis: Mock) -> None: + """A file-write failure after a successful export request is surfaced as a clean CLIError.""" + exported = _make_exported_execution() + api = mock_sync_apis.test_run_executions_api + api.export_test_run_execution_api_v1_test_run_executions__id__export_get.return_value = exported + + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + with patch("pathlib.Path.write_text", side_effect=OSError("Permission denied")): + result = cli_runner.invoke( + test_run_execution, ["export", "--id", "1", "--output-file", "/no/such/dir/out.json"] + ) + + assert result.exit_code == 1 + assert "Failed to write export file '/no/such/dir/out.json'" in result.output + assert "Permission denied" in result.output + + def test_export_timeout_error(self, cli_runner: CliRunner, mock_sync_apis: Mock) -> None: + """A timeout while exporting is surfaced as a clean, readable error instead of a raw + traceback (the original bug report for this command).""" + api = mock_sync_apis.test_run_executions_api + api.export_test_run_execution_api_v1_test_run_executions__id__export_get.side_effect = ( + ResponseHandlingException(ReadTimeout("timed out")) + ) + + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(test_run_execution, ["export", "--id", "1"]) + + assert result.exit_code == 1 + assert "Timed out waiting for the server" in result.output + + def test_export_help_message(self, cli_runner: CliRunner) -> None: + """Test the help message for the export command.""" + result = cli_runner.invoke(test_run_execution, ["export", "--help"]) + + assert result.exit_code == 0 + assert "--id" in result.output + assert "--output-file" in result.output + + +@pytest.mark.unit +@pytest.mark.cli +class TestImportExecutionCommand: + """Test cases for the `test-run-execution import` command.""" + + def test_import_success( + self, + cli_runner: CliRunner, + mock_sync_apis: Mock, + sample_test_run_execution: api_models.TestRunExecutionWithChildren, + temp_dir: Path, + ) -> None: + """A successful import reports the new execution's ID and title.""" + exported = _make_exported_execution(title="Imported Run") + import_file = temp_dir / "run.json" + import_file.write_text(exported.model_dump_json()) + + api = mock_sync_apis.test_run_executions_api + api.import_test_run_execution_api_v1_test_run_executions_import_post.return_value = sample_test_run_execution + + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(test_run_execution, ["import", "--file", str(import_file), "--project-id", "7"]) + + assert result.exit_code == 0 + assert f"imported as execution {sample_test_run_execution.id}" in result.output + call_args = api.import_test_run_execution_api_v1_test_run_executions_import_post.call_args + assert call_args.kwargs["project_id"] == 7 + assert call_args.kwargs["body"].import_file == import_file.read_bytes() + + def test_import_file_not_found(self, cli_runner: CliRunner, mock_sync_apis: Mock) -> None: + """A non-existent --file is rejected by Click's exists=True validation.""" + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke( + test_run_execution, ["import", "--file", "nonexistent.json", "--project-id", "1"] + ) + + assert result.exit_code == 2 + assert "does not exist" in result.output + + def test_import_db_revision_mismatch(self, cli_runner: CliRunner, mock_sync_apis: Mock, temp_dir: Path) -> None: + """A db_revision mismatch (422) is surfaced with the backend's plain-text detail.""" + exported = _make_exported_execution() + import_file = temp_dir / "run.json" + import_file.write_text(exported.model_dump_json()) + + api = mock_sync_apis.test_run_executions_api + api.import_test_run_execution_api_v1_test_run_executions_import_post.side_effect = UnexpectedResponse( + status_code=422, + content={"detail": "Mismatching 'db_revision'. Trying to import from abc123 to def456"}, + ) + + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(test_run_execution, ["import", "--file", str(import_file), "--project-id", "1"]) + + assert result.exit_code == 1 + assert "Mismatching 'db_revision'" in result.output + assert "{" not in result.output + + def test_import_requires_project_id(self, cli_runner: CliRunner, temp_dir: Path) -> None: + """The --project-id parameter is required.""" + import_file = temp_dir / "run.json" + import_file.write_text(_make_exported_execution().model_dump_json()) + + result = cli_runner.invoke(test_run_execution, ["import", "--file", str(import_file)]) + + assert result.exit_code != 0 + assert "Missing option" in result.output or "--project-id" in result.output + + def test_import_timeout_error(self, cli_runner: CliRunner, mock_sync_apis: Mock, temp_dir: Path) -> None: + """A timeout while importing is surfaced as a clean, readable error instead of a raw + traceback (the original bug report for this command).""" + import_file = temp_dir / "run.json" + import_file.write_text(_make_exported_execution().model_dump_json()) + + api = mock_sync_apis.test_run_executions_api + api.import_test_run_execution_api_v1_test_run_executions_import_post.side_effect = ( + ResponseHandlingException(ReadTimeout("timed out")) + ) + + with patch("th_cli.commands.test_run_execution.SyncApis", return_value=mock_sync_apis): + result = cli_runner.invoke(test_run_execution, ["import", "--file", str(import_file), "--project-id", "1"]) + + assert result.exit_code == 1 + assert "Timed out waiting for the server" in result.output + + def test_import_help_message(self, cli_runner: CliRunner) -> None: + """Test the help message for the import command.""" + result = cli_runner.invoke(test_run_execution, ["import", "--help"]) + + assert result.exit_code == 0 + assert "--file" in result.output + assert "--project-id" in result.output diff --git a/th_cli/api_lib_autogen/models.py b/th_cli/api_lib_autogen/models.py index 774f704..c030c82 100644 --- a/th_cli/api_lib_autogen/models.py +++ b/th_cli/api_lib_autogen/models.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: tmpktlfxzkc.json -# timestamp: 2026-09-01T11:49:15+00:00 +# filename: tmph7a30dit.json +# timestamp: 2026-09-10T23:33:56+00:00 from __future__ import annotations @@ -82,6 +82,7 @@ class PICSItem(BaseModel): class THConfig(BaseModel): prompt_timeout_seconds: Annotated[int | None, Field(title="Prompt Timeout Seconds")] = 60 enable_realtime_python_test_logs: Annotated[bool | None, Field(title="Enable Realtime Python Test Logs")] = None + enable_container_logs: Annotated[bool | None, Field(title="Enable Container Logs")] = None class PublicId(RootModel[str]): diff --git a/th_cli/client.py b/th_cli/client.py index 9c0af88..cc18bfa 100644 --- a/th_cli/client.py +++ b/th_cli/client.py @@ -13,15 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from httpx import Timeout + from th_cli.api_lib_autogen.api_client import ApiClient from th_cli.config import config from th_cli.exceptions import ConfigurationError -def get_client() -> ApiClient: - """Get API client with proper error handling.""" +def get_client(timeout: Timeout | float | None = None) -> ApiClient: + """Get API client with proper error handling. + + Args: + timeout: Optional httpx timeout to use for this client's requests, + forwarded to the underlying httpx.AsyncClient. Defaults to + httpx's own default if not provided. + """ try: - return ApiClient(host=f"http://{config.hostname}") + kwargs = {} if timeout is None else {"timeout": timeout} + return ApiClient(host=f"http://{config.hostname}", **kwargs) except Exception as e: raise ConfigurationError( f"Could not connect to API server at {config.hostname}. " diff --git a/th_cli/commands/test_run_execution.py b/th_cli/commands/test_run_execution.py index 8de203b..b807e15 100644 --- a/th_cli/commands/test_run_execution.py +++ b/th_cli/commands/test_run_execution.py @@ -13,20 +13,44 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import asyncio from contextlib import closing +from pathlib import Path import click +from httpx import Timeout, TimeoutException -from th_cli.api_lib_autogen.api_client import SyncApis -from th_cli.api_lib_autogen.exceptions import UnexpectedResponse +from th_cli.api_lib_autogen.api_client import AsyncApis, SyncApis +from th_cli.api_lib_autogen.exceptions import ResponseHandlingException, UnexpectedResponse +from th_cli.api_lib_autogen.models import ( + BodyImportTestRunExecutionApiV1TestRunExecutionsImportPost, + TestRunExecutionWithChildren, +) +from th_cli.async_cmd import async_cmd from th_cli.client import get_client -from th_cli.colorize import colorize_cmd_help, colorize_header, colorize_help, colorize_state, italic -from th_cli.exceptions import CLIError, handle_api_error +from th_cli.colorize import ( + colorize_cmd_help, + colorize_header, + colorize_help, + colorize_key_value, + colorize_state, + colorize_success, + italic, + set_colors_enabled, +) +from th_cli.exceptions import CLIError, handle_api_error, handle_file_error +from th_cli.test_run import logging as test_logging +from th_cli.test_run.websocket import TestRunSocket from th_cli.utils import __print_json table_format_header = "{:<6} {:<55} {}" table_format = "{:<6} {} {}" +# Repeating/exporting/importing a test run execution transfers its full config +# and logs, which can take significantly longer than httpx's 5s default read +# timeout for executions with a lot of log data. +TEST_RUN_EXECUTION_IO_TIMEOUT = Timeout(120.0, connect=10.0) # 120s total, 10s connect + _list_options = [ click.option( "--id", @@ -248,6 +272,116 @@ def pics_export(id: int, output_file: str) -> None: raise # Re-raise CLI Errors as-is +@test_run_execution.command( + name="repeat", + short_help=colorize_help("Repeat a test run execution"), + help=colorize_cmd_help("repeat", "Create a new execution with the same selected tests/config as an existing one"), +) +@click.option( + "--id", + "-i", + required=True, + type=int, + help=colorize_help("ID of the Test Run Execution to repeat"), +) +@click.option( + "--title", + "-n", + required=False, + type=str, + help=colorize_help( + "Title for the new execution. Defaults to the original title; the backend always " + "appends an updated timestamp regardless" + ), +) +@click.option( + "--no-color", + is_flag=True, + help=colorize_help("Disable colored output for test execution status."), +) +@click.option( + "--no-streaming", + is_flag=True, + help=colorize_help("Disable real-time log streaming via web browser (enabled by default)."), +) +@async_cmd +async def repeat(id: int, title: str | None, no_color: bool, no_streaming: bool) -> None: + if no_color: + set_colors_enabled(False) + + client = None + try: + client = get_client(timeout=TEST_RUN_EXECUTION_IO_TIMEOUT) + async_apis = AsyncApis(client) + new_execution = await __repeat_test_run_execution(async_apis, id, title) + await __start_and_stream_repeated_execution(async_apis, new_execution, enable_streaming=not no_streaming) + except CLIError: + raise # Re-raise CLI Errors as-is + except Exception as e: + raise CLIError(f"Unexpected error during repeated test execution: {e}") + finally: + if client: + await client.aclose() + + +@test_run_execution.command( + name="export", + short_help=colorize_help("Export a test run execution to a JSON file"), + help=colorize_cmd_help("export", "Export a test run execution's config and results to a JSON file"), +) +@click.option( + "--id", + "-i", + required=True, + type=int, + help=colorize_help("ID of the Test Run Execution to export"), +) +@click.option( + "--output-file", + "-o", + required=False, + type=click.Path(file_okay=True, dir_okay=False), + help=colorize_help("Output JSON file path (defaults to -execution.json)"), +) +def export(id: int, output_file: str | None) -> None: + try: + with closing(get_client(timeout=TEST_RUN_EXECUTION_IO_TIMEOUT)) as client: + sync_apis = SyncApis(client) + __export_test_run_execution(sync_apis, id, output_file) + + except CLIError: + raise # Re-raise CLI Errors as-is + + +@test_run_execution.command( + name="import", + short_help=colorize_help("Import a test run execution from a JSON file"), + help=colorize_cmd_help("import", "Import a test run execution previously exported with 'export'"), +) +@click.option( + "--file", + "-f", + required=True, + type=click.Path(file_okay=True, dir_okay=False, exists=True), + help=colorize_help("JSON file previously exported with 'test-run-execution export'"), +) +@click.option( + "--project-id", + "-p", + required=True, + type=int, + help=colorize_help("Project ID to import the execution into"), +) +def import_execution(file: str, project_id: int) -> None: + try: + with closing(get_client(timeout=TEST_RUN_EXECUTION_IO_TIMEOUT)) as client: + sync_apis = SyncApis(client) + __import_test_run_execution(sync_apis, file, project_id) + + except CLIError: + raise # Re-raise CLI Errors as-is + + def __test_run_execution_by_id(sync_apis: SyncApis, id: int, json: bool) -> None: try: test_run_execution_api = sync_apis.test_run_executions_api @@ -447,6 +581,164 @@ def __fetch_test_run_execution_pics_export(sync_apis: SyncApis, id: int, output_ handle_api_error(e, "fetch test run execution PICS export") +def _extract_error_detail(e: UnexpectedResponse) -> str: + """Pull a plain-text 'detail' message out of an UnexpectedResponse's content, if present.""" + content = e.content + if isinstance(content, bytes): + content = content.decode("utf-8", errors="ignore") + if isinstance(content, dict): + detail = content.get("detail") + if isinstance(detail, str): + return detail + return str(content) + + +def _timeout_or_connection_error(e: ResponseHandlingException, operation: str) -> str: + """Turn a low-level ResponseHandlingException into a readable CLI error message.""" + if isinstance(e.error, TimeoutException): + return ( + f"Timed out waiting for the server while trying to {operation} " + f"(waited {int(TEST_RUN_EXECUTION_IO_TIMEOUT.read)}s). Executions with a lot of " + "log data can take longer than that to transfer; please try again." + ) + return f"Could not {operation}: {e}. Please check if the API server is running and accessible." + + +async def __repeat_test_run_execution( + async_apis: AsyncApis, id: int, title: str | None +) -> TestRunExecutionWithChildren: + try: + test_run_execution_api = async_apis.test_run_executions_api + repeat_call = test_run_execution_api.repeat_test_run_execution_api_v1_test_run_executions__id__repeat_post + new_execution = await repeat_call(id=id, title=title) + click.echo( + colorize_success(f"Test run execution {id} repeated as new execution {new_execution.id}") + + f" ('{new_execution.title}')" + ) + return new_execution + except UnexpectedResponse as e: + if e.status_code == 404: + raise CLIError(f"Test run execution with ID '{id}' not found.") + handle_api_error(e, f"repeat test run execution '{id}'") + except ResponseHandlingException as e: + raise CLIError(_timeout_or_connection_error(e, f"repeat test run execution '{id}'")) + + +async def __start_and_stream_repeated_execution( + async_apis: AsyncApis, new_execution: TestRunExecutionWithChildren, enable_streaming: bool = True +) -> None: + """Start a repeated execution and attach to it the same way 'run-tests' does: + streaming live test progress and forwarding any user prompts to this terminal.""" + test_run_execution_api = async_apis.test_run_executions_api + + # Configure log output for this run before the websocket starts receiving log + # records. Without this, loguru's default stderr sink stays active and every + # raw log record gets printed straight to the terminal instead of being routed + # to the log file / streaming viewer, interleaving with the tree output below. + log_path = test_logging.configure_logger_for_run(title=new_execution.title, enable_log_streaming=enable_streaming) + test_logging.set_download_run_id(new_execution.id) + + header = colorize_header("Starting Test run") + title = colorize_key_value("Title", new_execution.title) + test_run_id = colorize_key_value("ID", str(new_execution.id)) + click.echo("") + click.echo(f"{header}:\n- {title}\n- {test_run_id}\n") + + log_stream_url = test_logging.get_log_stream_url() + if log_stream_url: + border = click.style("═" * 60, fg="cyan", bold=True) + click.echo(border) + click.echo(click.style(" 📋 Real-Time Log Viewer Available", fg="cyan", bold=True)) + click.echo(border) + click.echo(click.style(" View logs in real-time at:", fg="bright_white", bold=True)) + click.echo(" " + click.style(f"{log_stream_url}", fg="cyan", bold=True, underline=True)) + click.echo(click.style(" Logs will stream automatically as tests execute", fg="bright_white")) + click.echo(border) + click.echo("") + + socket = TestRunSocket(new_execution, project_config_dict=new_execution.execution_config or {}) + socket_task = asyncio.create_task(socket.connect_websocket()) + try: + try: + start_call = test_run_execution_api.start_test_run_execution_api_v1_test_run_executions__id__start_post + started_execution = await start_call(id=new_execution.id) + except UnexpectedResponse as e: + await _cancel_socket_task(socket_task) + if e.status_code == 409: + raise CLIError( + f"Execution {new_execution.id} was created but could not be started: " + f"{_extract_error_detail(e)}" + ) from e + handle_api_error(e, f"start repeated test run execution '{new_execution.id}'") + except ResponseHandlingException as e: + await _cancel_socket_task(socket_task) + raise CLIError( + _timeout_or_connection_error(e, f"start repeated test run execution '{new_execution.id}'") + ) from e + + socket.run = started_execution + await socket_task + click.echo(colorize_key_value("Log output in", italic(log_path))) + finally: + test_logging.stop_log_streaming() + + +async def _cancel_socket_task(socket_task: "asyncio.Task[None]") -> None: + """Cancel a pending websocket task and wait for it to finish unwinding.""" + socket_task.cancel() + await asyncio.gather(socket_task, return_exceptions=True) + + +def __export_test_run_execution(sync_apis: SyncApis, id: int, output_file: str | None) -> None: + try: + test_run_execution_api = sync_apis.test_run_executions_api + exported = test_run_execution_api.export_test_run_execution_api_v1_test_run_executions__id__export_get(id=id) + except UnexpectedResponse as e: + if e.status_code == 404: + raise CLIError(f"Test run execution with ID '{id}' not found.") + handle_api_error(e, f"export test run execution '{id}'") + except ResponseHandlingException as e: + raise CLIError(_timeout_or_connection_error(e, f"export test run execution '{id}'")) + + if not output_file: + if exported.test_run_execution.title: + import re + + output_file = re.sub(r"[^\w]", "", exported.test_run_execution.title) + "-execution.json" + else: + output_file = f"test_run_execution_{id}_export.json" + + try: + Path(output_file).write_text(exported.model_dump_json(indent=2), encoding="utf-8") + click.echo(colorize_success(f"Test run execution {id} exported to '{output_file}'")) + except OSError as e: + raise CLIError(f"Failed to write export file '{output_file}': {e}") + + +def __import_test_run_execution(sync_apis: SyncApis, file: str, project_id: int) -> None: + try: + file_bytes = Path(file).read_bytes() + except FileNotFoundError as e: + handle_file_error(e, "import file") + except OSError as e: + raise CLIError(f"Failed to read import file '{file}': {e}") + + body = BodyImportTestRunExecutionApiV1TestRunExecutionsImportPost(import_file=file_bytes) + + try: + test_run_execution_api = sync_apis.test_run_executions_api + response = test_run_execution_api.import_test_run_execution_api_v1_test_run_executions_import_post( + body=body, project_id=project_id + ) + click.echo( + colorize_success(f"Test run execution imported as execution {response.id}") + f" ('{response.title}')" + ) + except UnexpectedResponse as e: + handle_api_error(e, f"import test run execution from '{file}'") + except ResponseHandlingException as e: + raise CLIError(_timeout_or_connection_error(e, f"import test run execution from '{file}'")) + + def __print_table_test_executions(test_execution: list) -> None: __print_table_header() if isinstance(test_execution, list):