feat: web progress bar

src/imgtests/logger.py

@@ -1,16 +1,31 @@
+import json
 import logging
+import re
 import sys
-from typing import TYPE_CHECKING, Literal, Self, TextIO
+from enum import StrEnum
+from pathlib import Path
+from typing import Literal, Self, TextIO
 
 from pythonjsonlogger.json import JsonFormatter
 
-if TYPE_CHECKING:
-    from pathlib import Path
-
+from imgtests.constant import LIB_DATA_DIR
 
 LogLevel = Literal["debug", "info", "warning", "error", "critical"]
 
 
+class LoggingPatterns(StrEnum):
+    TASK_STARTED_STATUS = "RUNNING"
+    TASK_STATUS = r"Task id=([\w-]+) path=[\w|\.]+ state=(\w+)"
+    TESTS_COUNT = r"Total amount of tests per run: (\d+)"
+    RUNS_COUNT = r"Starting test run (\d+) of (\d+)"
+    DEFAULT_TEST_START = r"Starting '(.*\.)'.*"
+    DEFAULT_TEST_FINISH = r"'(.*\.)' test finished."
+    SUITE_START = r"Running suite (.*)\."
+    PROFILED_TEST_START = r"\[PLAN\] run stage=([\w-]+) tool=([\w-]+) subsystem=([\w-]+).*"
+    PROFILED_TEST_FINISH = r"\[PLAN\] done .*"
+    PROFILE_DONE = r"\[PROFILED\] DONE profile=(\w+) pattern=(\w+) .*"
+
+
 class StreamFormatter(logging.Formatter):
     def __init__(self: Self) -> None:
         self._level_fmt = "[%(levelname)s]"
@@ -23,6 +38,99 @@ def format(self: Self, record: logging.LogRecord) -> str:
         return super().format(record)
 
 
+class ProgressHandler(logging.Handler):
+    progress_template = {  # noqa: RUF012
+        "total_test_count": 0,
+        "test_count": 0,
+        "total_run_count": 0,
+        "current_test_run": 0,
+        "current_suite": "Not started yet",
+        "last_profile_done": "Not done yet",
+        "current_test": "Not started yet",
+    }
+
+    def __init__(self, level: logging._Level = logging.DEBUG):
+        super().__init__(level)
+        self.progress_data = {}
+        self.proc_to_task = {}
+
+    def emit(self, record: logging.LogRecord):  # noqa: PLR0915
+        proc = str(record.process)
+        msg = self.format(record)
+
+        # detect task started or finished
+        match = re.search(LoggingPatterns.TASK_STATUS, msg)
+        if match:
+            task_id = match.group(1)
+            status = match.group(2)
+            # task started
+            if status == LoggingPatterns.TASK_STARTED_STATUS:
+                self.proc_to_task[proc] = task_id
+            # task finished or broke
+            else:
+                self.proc_to_task[proc] = None
+            # flush progress_data for process
+            self.progress_data[proc] = self.progress_template.copy()
+
+        match = re.search(LoggingPatterns.TESTS_COUNT, msg)
+        if match and self.progress_data[proc]:
+            total = int(match.group(1))
+            self.progress_data[proc]["total_test_count"] = total
+
+        match = re.search(LoggingPatterns.RUNS_COUNT, msg)
+        if match and self.progress_data[proc]:
+            # set current run
+            cur = int(match.group(1))
+            total = int(match.group(2))
+            self.progress_data[proc]["current_test_run"] = cur
+            self.progress_data[proc]["total_run_count"] = total
+            # reset tests count
+            self.progress_data[proc]["test_count"] = 0
+
+        # default runner matches
+        match = re.search(LoggingPatterns.SUITE_START, msg)
+        if match and self.progress_data[proc]:
+            suite = match.group(1)
+            self.progress_data[proc]["current_suite"] = suite
+
+        match = re.search(LoggingPatterns.DEFAULT_TEST_START, msg)
+        if match and self.progress_data[proc]:
+            test = match.group(1)
+            self.progress_data[proc]["current_test"] = test
+
+        match = re.search(LoggingPatterns.DEFAULT_TEST_FINISH, msg)
+        if match and self.progress_data[proc]:
+            self.progress_data[proc]["test_count"] += 1
+            self.progress_data[proc]["current_test"] = "Not started yet"
+
+        # profiled runner matches
+        match = re.search(LoggingPatterns.PROFILE_DONE, msg)
+        if match and self.progress_data[proc]:
+            profile = "-".join([match.group(1), match.group(2)])
+            self.progress_data[proc]["last_profile_done"] = profile
+
+        match = re.search(LoggingPatterns.PROFILED_TEST_START, msg)
+        if match and self.progress_data[proc]:
+            subsystem = match.group(3)
+            profile = match.group(1)
+            tool = match.group(2)
+            self.progress_data[proc]["current_test"] = f"{subsystem}-{profile} via {tool}"
+
+        match = re.search(LoggingPatterns.PROFILED_TEST_FINISH, msg)
+        if match and self.progress_data[proc]:
+            self.progress_data[proc]["test_count"] += 1
+            self.progress_data[proc]["current_test"] = "Not started yet"
+
+        if proc in self.proc_to_task and self.proc_to_task[proc] is not None:
+            task_id = self.proc_to_task[proc]
+            with Path.open(
+                LIB_DATA_DIR / (task_id + "_progress.log"),
+                "w",
+                encoding="utf-8",
+            ) as file:
+                json.dump(self.progress_data[proc], file, indent=4)
+
+
 def set_handlers(
     logger: logging.Logger,
     filename: Path,
@@ -45,6 +153,7 @@ def set_handlers(
     """
     levelno = getattr(logging, log_level.upper())
     logger.setLevel(levelno)
+    logger.addHandler(__get_progress_handler())
     logger.addHandler(__get_file_handler(filename))
     if levelno in [logging.INFO, logging.DEBUG]:
         logger.addHandler(__get_stdout_handler())
@@ -82,3 +191,12 @@ def filter(self: Self, record: logging.LogRecord) -> bool:
     stdout_handler.setFormatter(StreamFormatter())
 
     return stdout_handler
+
+
+def __get_progress_handler() -> ProgressHandler:
+    progress_handle = ProgressHandler()
+    progress_handle.setLevel(logging.DEBUG)
+    progress_handle.setFormatter(StreamFormatter())
+    progress_handle.set_name("progress_handler")
+
+    return progress_handle

src/imgtests/runner.py

@@ -586,6 +586,11 @@ def run_tests(
     logger.info("Running tests for %s", distro)
     if mode == "default" and config is None:
         config = load_test_config(distro)
+
+    total_tests_amount = __calc_total_tests_amount(config, mode)
+    logger.info("Total amount of tests per run: %d", total_tests_amount)
+
+    # start test runs
     client = None
     match distro:
         case "yocto":
@@ -610,3 +615,26 @@ def get_test_name(
     if hasattr(test, "__class__"):
         return test.__class__.__name__
     return str(test)
+
+
+def __calc_total_tests_amount(config: dict[str, Any] | None, mode: Runner) -> int:
+    from imgtests.suites.map import ALL_SUITES  # noqa: PLC0415
+
+    total_tests_amount = 0
+    if mode == "default" and config:
+        for suite in config["suites"]:
+            if suite in config["selected_tests"]:
+                total_tests_amount += len(config["selected_tests"][suite])
+            else:
+                total_tests_amount += len(ALL_SUITES[suite].tests)
+            # default runner runs 2 system tests for each suite (runner.py: 616 -> 621 -> 149)
+            total_tests_amount += 2
+    if mode == "profiled":
+        tmp_config = build_profiled_settings(config=config)
+        total_tests_amount = len(tmp_config.subsystems)
+        # default profile config consists of 3 stages
+        if config is None:
+            total_tests_amount *= 3
+        if tmp_config.run_matrix:
+            total_tests_amount *= len(tmp_config.matrix_profiles)
+    return total_tests_amount

src/imgtests/web/locale/ru/LC_MESSAGES/django.po

@@ -119,6 +119,34 @@ msgid "Subsystems"
 msgstr "Подсистемы"
 
 #: tests_interface/templates/tests_interface/distro_page.html:135
+msgid "Progress"
+msgstr "Прогресс"
+
+#: tests_interface/templates/tests_interface/distro_page.html:140
+msgid "Test progress"
+msgstr "Прогресс по тестам"
+
+#: tests_interface/templates/tests_interface/distro_page.html:150
+msgid "Runs progress"
+msgstr "Прогресс по запускам:"
+
+#: tests_interface/templates/tests_interface/distro_page.html:159
+msgid "Current Suite:"
+msgstr "Текущий сценарий:"
+
+#: tests_interface/templates/tests_interface/distro_page.html:160
+msgid "Last Profile processed:"
+msgstr "Последный обработанный профиль:"
+
+#: tests_interface/templates/tests_interface/distro_page.html:161
+msgid "Current Test:"
+msgstr "Текущий тест:"
+
+#: tests_interface/templates/tests_interface/distro_page.html:164
+msgid "Progress data parsing error"
+msgstr "Ошибка парсинга данных прогресса"
+
+#: tests_interface/templates/tests_interface/distro_page.html:168
 msgid "Run tests"
 msgstr "Запустить тесты"
 

src/imgtests/web/locale/ru/LC_MESSAGES/djangojs.po

@@ -173,8 +173,8 @@ msgid "%(profile)s profile duration"
 msgstr "Длительность профиля %(profile)s"
 
 #: src/imgtests/web/static/js/launch_test.js:103
-#: src/imgtests/web/static/js/launch_test.js:137
-#: src/imgtests/web/static/js/launch_test.js:150
+#: src/imgtests/web/static/js/launch_test.js:146
+#: src/imgtests/web/static/js/launch_test.js:160
 msgid "Error: %(message)s"
 msgstr "Ошибка: %(message)s"
 
@@ -186,48 +186,60 @@ msgstr "Выполнение..."
 msgid "Tests running..."
 msgstr "Тесты выполняются..."
 
-#: src/imgtests/web/static/js/launch_test.js:130
+#: src/imgtests/web/static/js/launch_test.js:129
 msgid "Tests running... (Task ID: %(task_id)s)"
 msgstr "Тесты выполняются... (ID задачи: %(task_id)s)"
 
-#: src/imgtests/web/static/js/launch_test.js:140
+#: src/imgtests/web/static/js/launch_test.js:149
 msgid "Failed to start tests"
 msgstr "Не удалось запустить тесты"
 
-#: src/imgtests/web/static/js/launch_test.js:145
-#: src/imgtests/web/static/js/launch_test.js:155
+#: src/imgtests/web/static/js/launch_test.js:154
 #: src/imgtests/web/static/js/launch_test.js:165
+#: src/imgtests/web/static/js/launch_test.js:176
 msgid "Run tests"
 msgstr "Запустить тесты"
 
-#: src/imgtests/web/static/js/launch_test.js:174
+#: src/imgtests/web/static/js/launch_test.js:186
 msgid "Tests running... Please wait."
 msgstr "Тесты выполняются... Пожалуйста, подождите."
 
-#: src/imgtests/web/static/js/launch_test.js:180
+#: src/imgtests/web/static/js/launch_test.js:193
 msgid "Tests completed successfully."
 msgstr "Тесты завершены успешно."
 
-#: src/imgtests/web/static/js/launch_test.js:183
+#: src/imgtests/web/static/js/launch_test.js:196
 msgid "Test failed"
 msgstr "Тест не пройден"
 
-#: src/imgtests/web/static/js/launch_test.js:187
+#: src/imgtests/web/static/js/launch_test.js:200
 msgid "Error output:"
 msgstr "Вывод ошибок:"
 
-#: src/imgtests/web/static/js/launch_test.js:193
+#: src/imgtests/web/static/js/launch_test.js:206
 msgid "Output:"
 msgstr "Вывод:"
 
-#: src/imgtests/web/static/js/launch_test.js:198
+#: src/imgtests/web/static/js/launch_test.js:211
 msgid "Unknown status"
 msgstr "Неизвестный статус"
 
-#: src/imgtests/web/static/js/launch_test.js:204
+#: src/imgtests/web/static/js/launch_test.js:217
 msgid "Error checking status: %(message)s"
 msgstr "Ошибка проверки статуса: %(message)s"
 
+#: src/imgtests/web/static/js/launch_test.js:231
+msgid "Data load error"
+msgstr "Ошибка загрузки данных"
+
+#: src/imgtests/web/static/js/launch_test.js:257
+msgid "Run %(currentRun)s out of %(totalRuns)s is in progress (%(runsPercent)s%)"
+msgstr "Запуск %(currentRun)s из %(totalRuns)s в процессе (%(runsPercent)s%)"
+
+#: src/imgtests/web/static/js/launch_test.js:273
+msgid "JSON processing error:"
+msgstr "Ошибка обработки JSON"
+
 #: src/imgtests/web/static/js/test_config.js:22
 msgid "Failed to load configuration: %(message)s"
 msgstr "Не удалось загрузить конфигурацию: %(message)s"

src/imgtests/web/static/css/style.css

@@ -404,6 +404,59 @@ h1 i {
     margin: 0;
 }
 
+.progress-card {
+    background: white;
+    margin-bottom: 20px;
+    padding: 30px;
+    width: 100%;
+    border-radius: 12px;
+    border: 1px solid #ddd;
+}
+
+.progress-group {
+    margin-bottom: 20px;
+}
+
+.progress-label {
+    display: flex;
+    justify-content: space-between;
+    margin-bottom: 8px;
+    font-size: 14px;
+    font-weight: 600;
+    color: #7f8c8d;
+}
+
+.progress-bg {
+    background-color: #e0e0e0;
+    border-radius: 8px;
+    overflow: hidden;
+    height: 16px;
+    width: 100%;
+}
+
+.progress-bar {
+    background-color: #2ecc71;
+    height: 100%;
+    width: 0%;
+    transition: width 0.4s ease-out;
+}
+
+#runs-bar {
+    background-color: #3498db;
+}
+
+@keyframes pulse-animation {
+    0% { opacity: 1; }
+    50% { opacity: 0.4; }
+    100% { opacity: 1; }
+}
+
+.pulse {
+    animation: pulse-animation 1.5s infinite ease-in-out;
+    background-image: linear-gradient(45deg, rgba(255,255,255,.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,.15) 50%, rgba(255,255,255,.15) 75%, transparent 75%, transparent);
+    background-size: 1rem 1rem;
+}
+
 .duration-input-group {
     display: flex;
     flex-wrap: wrap;