From 57b1807e03a727a16299195f5d51fd0955efa7c9 Mon Sep 17 00:00:00 2001 From: Pranjal Date: Sun, 1 Mar 2026 07:42:14 +0530 Subject: [PATCH 01/19] Add external tutorial sources support (git/archive) for systemtests Allow tests.yaml entries to load tutorials from external git repos or archives. Rebased on develop and documented usage in the README. --- changelog-entries/732.md | 1 + tools/tests/README.md | 48 +++++- tools/tests/metadata_parser/metdata.py | 25 ++- tools/tests/systemtests/Systemtest.py | 180 +++++++++++++++++---- tools/tests/systemtests/TestSuite.py | 73 ++++++++- tools/tests/systemtests/sources.py | 206 +++++++++++++++++++++++++ 6 files changed, 482 insertions(+), 51 deletions(-) create mode 100644 changelog-entries/732.md create mode 100644 tools/tests/systemtests/sources.py diff --git a/changelog-entries/732.md b/changelog-entries/732.md new file mode 100644 index 000000000..a85a5a4b6 --- /dev/null +++ b/changelog-entries/732.md @@ -0,0 +1 @@ +- Added optional `source` blocks in `tests.yaml` so system tests can load tutorials from external git repositories or archives ([#732](https://github.com/precice/tutorials/pull/732)). diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..d3debe79a 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -36,7 +36,7 @@ Workflow for the preCICE v3 release testing: 6. Download the build artifacts from Summary > runs. - - In there, you may want to check the `stdout.log` and `stderr.log` files. + - In there, you may want to check the `system-tests-stdout.log` and `system-tests-stderr.log` files. - The produced results are in `precice-exports/`, the reference results in `reference-results-unpacked`. - Compare using, e.g., ParaView or [fieldcompare](https://gitlab.com/dglaeser/fieldcompare): `fieldcompare dir precice-exports/ reference/`. The `--diff` option will give you `precice-exports/diff_*.vtu` files, while you can also try different tolerances with `-rtol` and `-atol`. @@ -105,7 +105,9 @@ In this case, building and running seems to work out, but the tests fail because The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. -Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. +Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: `system-tests-stderr.log` and `system-tests-stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files under `precice-exports/`; if the comparison fails, those files are copied into a `diff-results/` subfolder in the same run directory (mirroring any subpaths under `precice-exports/`) so you can open them (e.g. in ParaView) to see where results differ from the reference. On successful comparisons, `diff-results/` is therefore absent. + +For implicit-coupling runs, `precice-*-iterations.log` files are collected into `iterations-logs/` and compared by hash against archived reference copies (stored next to each reference `.tar.gz` in a `*.iterations-logs/` directory, or legacy `.iterations-hashes.json` sidecars). A mismatch fails the test. ## Adding new tests @@ -118,6 +120,8 @@ In order for the systemtests to pick up the tutorial we need to define a `metada To add a testsuite just open the `tests.yaml` file and use the output of `python print_case_combinations.py` to add the right case combinations you want to test. Note that you can specify a `reference_result` which is not yet present. The `generate_reference_data.py` will pick that up and create it for you. Note that its important to carefully check the paths of the `reference_result` in order to not have typos in there. Also note that same cases in different testsuites should use the same `reference_result`. +To cap the preCICE simulation time for a specific test without editing `precice-config.xml`, add an optional `max_time` (positive float, overrides ``) or `max_time_windows` (positive integer, overrides ``) field to the tutorial entry. Applies to both test runs and reference result generation. + ### Generate reference results Since we need data to compare against, you need to run `python generate_reference_data.py`. This process might take a while. @@ -167,7 +171,7 @@ Metadata and workflow/script files: - ... - `dockerfiles/` - Multi-stage build Dockerfiles that define how to build each component, in a layered approach - - `docker-compose.template.yaml`: Describes how to prepare each test (Docker Componse service template) + - `docker-compose.template.yaml`: Describes how to prepare each test (Docker Compose service template) - `docker-compose.field_compare.template.yaml`: Describes how to compare results with fieldcompare (Docker Compose service template) - `components.yaml`: Declares the available components and their parameters/options - `reference_results.metadata.template`: Template for reporting the versions used to generate the reference results @@ -231,9 +235,9 @@ cases: Description: - `name`: A human-readable, descriptive name -- `path`: Where the tutorial is located, relative to the tutorials repository +- `path`: Where the tutorial is located, relative to the tutorials repository (or the tutorial folder name for external sources) - `url`: A web page with more information on the tutorial -- `participants`: A list of preCICE participants, typically corresponing to different domains of the simulation +- `participants`: A list of preCICE participants, typically corresponding to different domains of the simulation - `cases`: A list of solver configuration directories. Each element of the list includes: - `participant`: Which participant this solver case can serve as - `directory`: Where the case directory is located, relative to the tutorial directory @@ -279,7 +283,9 @@ This `openfoam-adapter` component has the following attributes: Since the docker containers are still a bit mixed in terms of capabilities and support for different build_argument combinations the following rules apply: -- A build_argument ending in **_REF** means that it refers to a git commit-ish (like a tag or commit) beeing used to build the image. Its important to not use branch names here as we heavily rely on dockers build cache to speedup things. But since the input variable to the docker builder will not change, we might have wrong cache hits. +- A build argument ending in `_REF` refers to a git commit-ish (like a tag or commit) being used to build the image. It is important to not use branch names here as we heavily rely on Docker's build cache to speedup things. But since the input variable to the docker builder will not change, we might have wrong cache hits. +- Some workflows set variables ending in `_PR`. These specify the GitHub pull request which provides the above `_REF` and can be on a fork. +- A build argument ending in `_VERSION` refers to the version of a third-party dependency to use (e.g., DUNE). - All other build_arguments are free of rules and up to the container maintainer. ### Component templates @@ -331,15 +337,43 @@ test_suites: - fluid-openfoam - solid-fenics reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz + timeout: 1200 ``` +The optional `timeout` field (in seconds) sets the maximum time for the solver run and fieldcompare phases of that specific case. If omitted, it defaults to `GLOBAL_TIMEOUT` (currently 900s, overridable via the `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable). + +#### External tutorial sources + +By default, every `path` must exist in the local `precice/tutorials` checkout. For tutorials maintained elsewhere, add an optional `source` block: + +```yaml +- path: flow-over-heated-plate + source: + type: git + url: https://github.com/precice/tutorials.git + ref: develop + subdir: . # optional subdirectory inside the repository + case_combination: + - fluid-openfoam + - solid-openfoam + reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz +``` + +Supported `type` values: + +- `local` (default): use the tutorial from this repository. +- `git`: shallow-clone `url` at `ref` (cached under `~/.cache/precice-tutorials` or `PRECICE_EXTERNAL_CACHE_DIR`). +- `archive`: download and extract a `.tar.gz` / `.zip` from `url`. + +The runner copies the resolved tutorial into the run directory, then continues with the usual Docker build/run and fieldcompare steps. `TUTORIALS_REF` / `TUTORIALS_PR` build arguments still apply only to **local** tutorials; external tutorials are pinned by `source.ref`. + This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run. ### Generate Reference Results #### via GitHub workflow (recommended) -The preferred way of adding reference results is via the manual triggerable `Generate reference results (manual)` workflow. This takes two inputs: +The preferred way of adding reference results is via the manual `Generate reference results (manual)` workflow. This takes two inputs: - `from_ref`: branch where the new test configuration (e.g added tests, new reference_versions.yaml) is - `commit_msg`: commit message for adding the reference results into the branch diff --git a/tools/tests/metadata_parser/metdata.py b/tools/tests/metadata_parser/metdata.py index 75b5bb425..7bb9863b5 100644 --- a/tools/tests/metadata_parser/metdata.py +++ b/tools/tests/metadata_parser/metdata.py @@ -6,6 +6,9 @@ import itertools from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR +# Import TutorialSource from systemtests.sources (used for external tutorial sources). +from systemtests.sources import TutorialSource + @dataclass class BuildArgument: @@ -279,13 +282,15 @@ def from_cases_tuple(cls, cases: Tuple[Case], tutorial: Tutorial): class ReferenceResult: path: Path case_combination: CaseCombination + base_dir: Optional[Path] = None def __repr__(self) -> str: return f"{self.path.as_posix()}" def __post_init__(self): # built full path - self.path = PRECICE_TUTORIAL_DIR / self.path + base = self.base_dir if self.base_dir is not None else PRECICE_TUTORIAL_DIR + self.path = Path(base) / self.path @dataclass @@ -299,6 +304,7 @@ class Tutorial: url: str participants: List[str] cases: List[Case] + source: "TutorialSource" = field(default_factory=TutorialSource.local) case_combinations: List[CaseCombination] = field(init=False) def __post_init__(self): @@ -355,13 +361,16 @@ def get_case_by_string(self, case_name: str) -> Optional[Case]: return None @classmethod - def from_yaml(cls, path, available_components): + def from_yaml(cls, path, available_components, base_dir=None, source=None): """ Creates a Tutorial instance from a YAML file. Args: - path: The path to the YAML file. + path: The path to the metadata.yaml file. available_components: The Components instance containing available components. + base_dir: Optional base directory for resolving tutorial path (for external sources). + Defaults to PRECICE_TUTORIAL_DIR. + source: Optional TutorialSource (for external tutorials). Returns: An instance of Tutorial. @@ -369,7 +378,8 @@ def from_yaml(cls, path, available_components): with open(path, 'r') as f: data = yaml.safe_load(f) name = data['name'] - path = PRECICE_TUTORIAL_DIR / data['path'] + base = base_dir if base_dir is not None else PRECICE_TUTORIAL_DIR + tutorial_path = Path(base) / data['path'] url = data['url'] participants = data.get('participants', []) cases_raw = data.get('cases', {}) @@ -377,7 +387,10 @@ def from_yaml(cls, path, available_components): for case_name in cases_raw.keys(): cases.append(Case.from_dict( case_name, cases_raw[case_name], available_components)) - return cls(name, path, url, participants, cases) + tut = cls(name, tutorial_path, url, participants, cases) + if source is not None: + tut.source = source + return tut class Tutorials(list): @@ -440,4 +453,4 @@ def from_path(cls, path): for yaml_path in yaml_files: tut = Tutorial.from_yaml(yaml_path, available_components) tutorials.append(tut) - return cls(tutorials) + return cls(tutorials) \ No newline at end of file diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index bfb1151cf..65d21f916 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -1,5 +1,6 @@ import subprocess -from typing import List, Dict, Optional +from .sources import resolve_tutorial_root, PRECICE_EXTERNAL_CACHE_DIR +from typing import List, Dict, Optional, Tuple from jinja2 import Environment, FileSystemLoader from dataclasses import dataclass, field import shutil @@ -19,9 +20,11 @@ import os -GLOBAL_TIMEOUT = 600 +GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 900)) SHORT_TIMEOUT = 10 +DIFF_RESULTS_DIR = "diff-results" + def slugify(value, allow_unicode=False): """ @@ -117,7 +120,7 @@ def _get_length_of_name(results: List[SystemtestResult]) -> int: with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: print("\n\n", file=f) print( - "In case a test fails, download the archive from the bottom of this page and look into each `stdout.log` and `stderr.log`. The time spent in each step might already give useful hints.", + "In case a test fails, download the archive from the bottom of this page and look into each `system-tests-stdout.log` and `system-tests-stderr.log`. The time spent in each step might already give useful hints.", file=f) print( "See the [documentation](https://precice.org/dev-docs-system-tests.html#understanding-what-went-wrong).", @@ -134,6 +137,9 @@ class Systemtest: arguments: SystemtestArguments case_combination: CaseCombination reference_result: ReferenceResult + max_time: float | None = None + max_time_windows: int | None = None + timeout: int = GLOBAL_TIMEOUT params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -299,24 +305,34 @@ def __copy_tutorial_into_directory(self, run_directory: Path): """ current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.run_directory = run_directory - pr_requested = self.params_to_use.get("TUTORIALS_PR") - if pr_requested: - logging.debug(f"Fetching the PR {pr_requested} HEAD reference") - self._fetch_pr(PRECICE_TUTORIAL_DIR, pr_requested) - current_ref = self._get_git_ref(PRECICE_TUTORIAL_DIR) - ref_requested = self.params_to_use.get("TUTORIALS_REF") - if ref_requested: - logging.debug(f"Checking out tutorials {ref_requested} before copying") - self._fetch_ref(PRECICE_TUTORIAL_DIR, ref_requested) - self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, ref_requested) - - self.tutorial_folder = slugify(f'{self.tutorial.path.name}_{self.case_combination.cases}_{current_time_string}') + current_ref = None + ref_requested = None + + if self.tutorial.source.type == "local": + pr_requested = self.params_to_use.get("TUTORIALS_PR") + if pr_requested: + logging.debug(f"Fetching the PR {pr_requested} HEAD reference") + self._fetch_pr(PRECICE_TUTORIAL_DIR, pr_requested) + current_ref = self._get_git_ref(PRECICE_TUTORIAL_DIR) + ref_requested = self.params_to_use.get("TUTORIALS_REF") + if ref_requested: + logging.debug(f"Checking out tutorials {ref_requested} before copying") + self._fetch_ref(PRECICE_TUTORIAL_DIR, ref_requested) + self._checkout_ref_in_subfolder( + PRECICE_TUTORIAL_DIR, self.tutorial.path, ref_requested) + + self.tutorial_folder = slugify( + f'{self.tutorial.path.name}_{self.case_combination.cases}_{current_time_string}') destination = run_directory / self.tutorial_folder - src = self.tutorial.path + src = resolve_tutorial_root( + self.tutorial.path, + self.tutorial.source, + PRECICE_EXTERNAL_CACHE_DIR, + ) self.system_test_dir = destination shutil.copytree(src, destination) - if ref_requested: + if self.tutorial.source.type == "local" and ref_requested: with open(destination / "tutorials_ref", 'w') as file: file.write(ref_requested) self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, current_ref) @@ -353,26 +369,59 @@ def __write_env_file(self): for key, value in self.env.items(): env_file.write(f"{key}={value}\n") - def __unpack_reference_results(self): - with tarfile.open(self.reference_result.path) as reference_results_tared: - # specify which folder to extract to - reference_results_tared.extractall(self.system_test_dir / PRECICE_REL_REFERENCE_DIR) - logging.debug( - f"extracting {self.reference_result.path} into {self.system_test_dir / PRECICE_REL_REFERENCE_DIR}") + def __unpack_reference_results(self) -> Tuple[bool, str]: + if not self.reference_result.path.exists(): + error_message = ( + f"Reference results archive was not found for {self}. " + f"Expected file: {self.reference_result.path}. " + "Please generate the reference results first or update tests.yaml accordingly.") + logging.error(error_message) + return False, error_message + + try: + # Base directory where reference results should be extracted + dest_dir = self.system_test_dir / PRECICE_REL_REFERENCE_DIR + dest_dir.mkdir(parents=True, exist_ok=True) + dest_dir_resolved = dest_dir.resolve() + + with tarfile.open(self.reference_result.path) as reference_results_tared: + # Validate that each member will be extracted within dest_dir + for member in reference_results_tared.getmembers(): + member_path = dest_dir / member.name + member_path_resolved = member_path.resolve() + # Ensure the resolved member path is within the destination directory + if os.path.commonpath([str(dest_dir_resolved), str( + member_path_resolved)]) != str(dest_dir_resolved): + logging.error( + f"Unsafe path detected in reference results archive {self.reference_result.path} " + f"for {self}: {member.name}") + return False + + # All paths are safe; extract into the destination directory + reference_results_tared.extractall(dest_dir) + + logging.debug( + f"extracting {self.reference_result.path} into {dest_dir}") + return True, "" + except (tarfile.TarError, OSError) as e: + error_message = ( + f"Could not unpack reference results archive {self.reference_result.path} for {self}: {e}") + logging.error(error_message) + return False, error_message def _run_field_compare(self): """ - Writes the Docker Compose file to disk, executes docker-compose up, and handles the process output. - - Args: - docker_compose_content: The content of the Docker Compose file. + Executes the field comparison step after unpacking reference results. Returns: - A SystemtestResult object containing the state. + A FieldCompareResult object containing the command outcome and logs. """ logging.debug(f"Running fieldcompare for {self}") time_start = time.perf_counter() - self.__unpack_reference_results() + unpack_success, unpack_error_message = self.__unpack_reference_results() + if not unpack_success: + elapsed_time = time.perf_counter() - time_start + return FieldCompareResult(1, [], [unpack_error_message], self, elapsed_time) docker_compose_content = self.__get_field_compare_compose_file() stdout_data = [] stderr_data = [] @@ -394,7 +443,7 @@ def _run_field_compare(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() raise KeyboardInterrupt from k @@ -413,6 +462,46 @@ def _run_field_compare(self): elapsed_time = time.perf_counter() - time_start return FieldCompareResult(1, stdout_data, stderr_data, self, elapsed_time) + def __archive_fieldcompare_diffs(self) -> None: + """ + Copy fieldcompare diff VTK files from precice-exports/ into diff-results/, + preserving paths under precice-exports/ so nested outputs are not skipped + and identical basenames in different folders do not overwrite each other. + """ + exports_dir = self.system_test_dir / PRECICE_REL_OUTPUT_DIR + if not exports_dir.is_dir(): + return + suffixes = (".vtu", ".vtk", ".vtp") + dest_root = self.system_test_dir / DIFF_RESULTS_DIR + seen_resolved: set[Path] = set() + archived_count = 0 + for path in exports_dir.rglob("*"): + if not path.is_file(): + continue + if path.suffix.lower() not in suffixes: + continue + if "diff" not in path.name.lower(): + continue + resolved = path.resolve() + if resolved in seen_resolved: + continue + try: + rel = path.relative_to(exports_dir) + except ValueError: + continue + seen_resolved.add(resolved) + dest_path = dest_root / rel + dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, dest_path) + archived_count += 1 + if archived_count: + logging.debug( + "Archived %d fieldcompare diff file(s) to %s for %s", + archived_count, + dest_root, + self, + ) + def _build_docker(self): """ Builds the docker image @@ -483,7 +572,7 @@ def _run_tutorial(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) @@ -508,16 +597,40 @@ def __repr__(self): return f"{self.tutorial.name} {self.case_combination}" def __write_logs(self, stdout_data: List[str], stderr_data: List[str]): - with open(self.system_test_dir / "stdout.log", 'w') as stdout_file: + with open(self.system_test_dir / "system-tests-stdout.log", 'w') as stdout_file: stdout_file.write("\n".join(stdout_data)) - with open(self.system_test_dir / "stderr.log", 'w') as stderr_file: + with open(self.system_test_dir / "system-tests-stderr.log", 'w') as stderr_file: stderr_file.write("\n".join(stderr_data)) + def __apply_max_time_override(self): + """Overwrite or value in precice-config.xml.""" + if self.max_time is None and self.max_time_windows is None: + return + config_path = self.system_test_dir / "precice-config.xml" + text = config_path.read_text() + new_text = text + if self.max_time is not None: + pattern = r'({self.max_time}\2', new_text) + if count == 0: + logging.warning(f"No tag found in {config_path}") + else: + logging.info(f"Overwrote to {self.max_time} in {config_path}") + if self.max_time_windows is not None: + pattern = r'({self.max_time_windows}\2', new_text) + if count == 0: + logging.warning(f"No tag found in {config_path}") + else: + logging.info(f"Overwrote to {self.max_time_windows} in {config_path}") + config_path.write_text(new_text) + def __prepare_for_run(self, run_directory: Path): """ Prepares the run_directory with folders and datastructures needed for every systemtest execution """ self.__copy_tutorial_into_directory(run_directory) + self.__apply_max_time_override() self.__copy_tools(run_directory) self.__put_gitignore(run_directory) host_uid, host_gid = self.__get_uid_gid() @@ -566,6 +679,7 @@ def run(self, run_directory: Path): std_out.extend(fieldcompare_result.stdout_data) std_err.extend(fieldcompare_result.stderr_data) if fieldcompare_result.exit_code != 0: + self.__archive_fieldcompare_diffs() self.__write_logs(std_out, std_err) logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") return SystemtestResult( diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 9d8c2ac72..7bfacb81d 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -1,6 +1,20 @@ from dataclasses import dataclass, field +from pathlib import Path from typing import Optional, List, Dict -from metadata_parser.metdata import Tutorials, Tutorial, Case, CaseCombination, ReferenceResult +from metadata_parser.metdata import ( + Tutorials, + Tutorial, + Case, + CaseCombination, + ReferenceResult, + Components, +) +from paths import PRECICE_TESTS_DIR +from systemtests.sources import ( + TutorialSource, + resolve_tutorial_root, + PRECICE_EXTERNAL_CACHE_DIR, +) import yaml @@ -10,6 +24,9 @@ class TestSuite: name: str cases_of_tutorial: Dict[Tutorial, List[CaseCombination]] reference_results: Dict[Tutorial, List[ReferenceResult]] + max_times: Dict[Tutorial, list] = field(default_factory=dict) + max_time_windows: Dict[Tutorial, list] = field(default_factory=dict) + timeouts: Dict[Tutorial, List] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -42,35 +59,81 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): An instance of TestSuites. """ testsuites = [] + available_components = Components.from_yaml(PRECICE_TESTS_DIR / "components.yaml") with open(path, 'r') as f: data = yaml.safe_load(f) test_suites_raw = data['test_suites'] for test_suite_name in test_suites_raw: case_combinations_of_tutorial = {} reference_results_of_tutorial = {} - # iterate over tutorials: + max_times_of_tutorial = {} + max_time_windows_of_tutorial = {} + timeouts_of_tutorial = {} for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: + source = TutorialSource.from_dict(tutorial_case.get('source')) tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) + if not tutorial and source.type != "local": + tutorial_root = resolve_tutorial_root( + Path(tutorial_case['path']), + source, + PRECICE_EXTERNAL_CACHE_DIR, + ) + metadata_path = tutorial_root / "metadata.yaml" + if not metadata_path.exists(): + raise FileNotFoundError( + f"No metadata.yaml found for external tutorial " + f"{tutorial_case['path']} at {tutorial_root}" + ) + tutorial = Tutorial.from_yaml( + metadata_path, + available_components, + base_dir=tutorial_root.parent, + source=source, + ) + parsed_tutorials.tutorials.append(tutorial) if not tutorial: raise Exception(f"No tutorial with path {tutorial_case['path']} found.") - # initialize the datastructure for the new Testsuite if tutorial not in case_combinations_of_tutorial: case_combinations_of_tutorial[tutorial] = [] reference_results_of_tutorial[tutorial] = [] + max_times_of_tutorial[tutorial] = [] + max_time_windows_of_tutorial[tutorial] = [] + timeouts_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( tutorial_case['case_combination'], tutorial) if case_combination_requested in all_case_combinations: case_combinations_of_tutorial[tutorial].append(case_combination_requested) + ref_base = tutorial.path.parent if source.type != "local" else None reference_results_of_tutorial[tutorial].append(ReferenceResult( - tutorial_case['reference_result'], case_combination_requested)) + tutorial_case['reference_result'], + case_combination_requested, + base_dir=ref_base, + )) + max_time_raw = tutorial_case.get('max_time', None) + if max_time_raw is not None and (not isinstance( + max_time_raw, (int, float)) or max_time_raw <= 0): + raise ValueError(f"max_time must be a positive number, got {max_time_raw!r}") + max_times_of_tutorial[tutorial].append(max_time_raw) + mtw_raw = tutorial_case.get('max_time_windows', None) + if mtw_raw is not None and (not isinstance(mtw_raw, int) or mtw_raw <= 0): + raise ValueError(f"max_time_windows must be a positive integer, got {mtw_raw!r}") + max_time_windows_of_tutorial[tutorial].append(mtw_raw) + + timeout_value = tutorial_case.get('timeout', None) + if timeout_value is not None and not isinstance(timeout_value, int): + raise TypeError( + f"Expected 'timeout' to be an integer or None, but got {type(timeout_value).__name__} " + f"(value: {timeout_value}) in tutorial '{tutorial}'." + ) + timeouts_of_tutorial[tutorial].append(timeout_value) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial)) + reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial)) return cls(testsuites) diff --git a/tools/tests/systemtests/sources.py b/tools/tests/systemtests/sources.py new file mode 100644 index 000000000..7f4e6ad92 --- /dev/null +++ b/tools/tests/systemtests/sources.py @@ -0,0 +1,206 @@ +""" +Support for external tutorial sources (git, archive) in systemtests. +""" + +import hashlib +import logging +import os +import shutil +import subprocess +import tarfile +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +# Cache directory for fetched tutorials. Can be overridden via PRECICE_EXTERNAL_CACHE_DIR env. +_DEFAULT_CACHE = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "precice-tutorials" +PRECICE_EXTERNAL_CACHE_DIR = Path(os.environ.get("PRECICE_EXTERNAL_CACHE_DIR", _DEFAULT_CACHE)) + + +@dataclass +class TutorialSource: + """Describes where a tutorial is sourced from.""" + + type: str # "local" | "git" | "archive" + url: Optional[str] = None + ref: Optional[str] = None + subdir: Optional[str] = None + + @classmethod + def local(cls) -> "TutorialSource": + return cls(type="local") + + @classmethod + def from_dict(cls, data: dict) -> "TutorialSource": + if data is None or data.get("type") == "local": + return cls.local() + return cls( + type=data["type"], + url=data.get("url"), + ref=data.get("ref"), + subdir=data.get("subdir"), + ) + + +def _cache_key(prefix: str, url: str, ref: Optional[str] = None, subdir: Optional[str] = None) -> str: + """Generate a short content-addressable cache key.""" + parts = [url] + if ref: + parts.append(ref) + if subdir: + parts.append(subdir) + raw = f"{prefix}:{':'.join(parts)}" + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + +def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: Optional[str] = None) -> Path: + """ + Clone or update a git repository and return the path to the checkout. + If subdir is given, returns the path to that subdirectory within the repo. + """ + cache_dir.mkdir(parents=True, exist_ok=True) + key = _cache_key("git", url, ref, subdir) + checkout = cache_dir / key + + if checkout.exists(): + try: + subprocess.run( + ["git", "-C", str(checkout), "fetch", "origin", ref, "--depth", "1"], + check=True, + capture_output=True, + timeout=120, + ) + subprocess.run( + ["git", "-C", str(checkout), "checkout", "FETCH_HEAD"], + check=True, + capture_output=True, + timeout=60, + ) + except subprocess.CalledProcessError as e: + logging.warning(f"Git fetch/checkout failed for {url}, recloning: {e}") + shutil.rmtree(checkout, ignore_errors=True) + + if not checkout.exists(): + result = subprocess.run( + ["git", "clone", "--depth", "1", "--branch", ref, url, str(checkout)], + capture_output=True, + text=True, + timeout=300, + ) + if result.returncode != 0: + # Fallback: branch may not exist (e.g. repo uses develop/master instead of main) + # Clone without branch, then fetch and checkout ref (with common aliases) + shutil.rmtree(checkout, ignore_errors=True) + logging.debug( + f"git clone --branch {ref} failed ({result.stderr}), trying clone + fetch" + ) + subprocess.run( + ["git", "clone", "--depth", "1", url, str(checkout)], + check=True, + capture_output=True, + timeout=300, + ) + refs_to_try = [ref] + if ref == "main": + refs_to_try.extend(["develop", "master"]) + elif ref == "master": + refs_to_try.extend(["main", "develop"]) + last_err = None + for r in refs_to_try: + res = subprocess.run( + ["git", "-C", str(checkout), "fetch", "origin", r, "--depth", "1"], + capture_output=True, + text=True, + timeout=120, + ) + if res.returncode == 0: + subprocess.run( + ["git", "-C", str(checkout), "checkout", "FETCH_HEAD"], + check=True, + capture_output=True, + timeout=60, + ) + break + last_err = res.stderr + else: + raise RuntimeError( + f"Could not fetch ref '{ref}' (tried {refs_to_try}): {last_err}" + ) + + if subdir: + subpath = checkout / subdir + if not subpath.is_dir(): + raise FileNotFoundError(f"Subdirectory {subdir} not found in {url} (ref {ref})") + return subpath + return checkout + + +def fetch_archive(url: str, cache_dir: Path, subdir: Optional[str] = None) -> Path: + """ + Download and extract an archive (tar.gz, tar, zip) and return the path. + """ + import urllib.request + + cache_dir.mkdir(parents=True, exist_ok=True) + key = _cache_key("archive", url, subdir=subdir) + extract_dir = cache_dir / key + + if extract_dir.exists(): + return extract_dir / subdir if subdir else extract_dir + + with tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") as tmp: + tmp_path = Path(tmp.name) + try: + logging.info(f"Downloading {url}") + urllib.request.urlretrieve(url, tmp_path) + + extract_dir.mkdir(parents=True, exist_ok=True) + if url.endswith(".tar.gz") or url.endswith(".tgz") or url.endswith(".tar"): + with tarfile.open(tmp_path, "r:*") as tf: + tf.extractall(extract_dir) + else: + import zipfile + + with zipfile.ZipFile(tmp_path, "r") as zf: + zf.extractall(extract_dir) + finally: + tmp_path.unlink(missing_ok=True) + + if subdir: + subpath = extract_dir / subdir + if not subpath.is_dir(): + raise FileNotFoundError(f"Subdirectory {subdir} not found in {url}") + return subpath + return extract_dir + + +def resolve_tutorial_root( + path: Path, + source: TutorialSource, + cache_dir: Path, +) -> Path: + """ + Resolve the filesystem path to the tutorial root. + + For local sources, returns path as-is (already under PRECICE_TUTORIAL_DIR). + For git/archive sources, fetches the repository/archive and returns the path + to the tutorial directory. The tutorial name (path.name) is used as the + subdirectory within the fetched content. + """ + if source.type == "local": + return path + + if source.type == "git": + if not source.url or not source.ref: + raise ValueError("git source requires 'url' and 'ref'") + root = fetch_git_repo(source.url, source.ref, cache_dir, source.subdir) + return root / path.name + + if source.type == "archive": + if not source.url: + raise ValueError("archive source requires 'url'") + root = fetch_archive(source.url, cache_dir, source.subdir) + return root / path.name + + raise ValueError(f"Unknown source type: {source.type}") From 4d68a311db36d0f2ad10e62ce8779abb86dc9ee2 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Mon, 18 May 2026 17:44:54 +0530 Subject: [PATCH 02/19] Fix pre-commit style checks for PR #732 --- tools/tests/README.md | 1 - tools/tests/metadata_parser/metdata.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/tests/README.md b/tools/tests/README.md index 7a0c02199..3d3a96abb 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -340,7 +340,6 @@ test_suites: The optional `timeout` field (in seconds) sets the maximum time for the solver run and fieldcompare phases of that specific case. If omitted, it defaults to `GLOBAL_TIMEOUT` (currently 900s, overridable via the `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable). - #### External tutorial sources By default, every `path` must exist in the local `precice/tutorials` checkout. For tutorials maintained elsewhere, add an optional `source` block: diff --git a/tools/tests/metadata_parser/metdata.py b/tools/tests/metadata_parser/metdata.py index 7bb9863b5..585fc9963 100644 --- a/tools/tests/metadata_parser/metdata.py +++ b/tools/tests/metadata_parser/metdata.py @@ -453,4 +453,4 @@ def from_path(cls, path): for yaml_path in yaml_files: tut = Tutorial.from_yaml(yaml_path, available_components) tutorials.append(tut) - return cls(tutorials) \ No newline at end of file + return cls(tutorials) From a9e6c785e35268365c06878b4789a3315a2b2812 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 24 Jun 2026 17:03:38 +0200 Subject: [PATCH 03/19] Add a concrete example in the tests.yaml --- tools/tests/tests.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 4746de69f..853b968cd 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -763,3 +763,26 @@ test_suites: selected: tutorials: - *elastic-tube-1d_fluid-python_solid-python + + external: + tutorials: + - path: partitioned-heat-conduction-robin + source: + type: git + url: https://github.com/vidulejs/tutorials.git + ref: partitioned-heat-conduction-mixed-robin + subdir: . + case_combination: + - left-openfoam + - right-openfoam + reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz + - path: partitioned-heat-conduction-mixedbc + source: + type: git + url: https://github.com/vidulejs/tutorials.git + ref: partitioned-heat-conduction-mixed-robin + subdir: . + case_combination: + - left-openfoam + - right-openfoam + reference_result: ./partitioned-heat-conduction-mixedbc/reference_results/left-openfoam_right-openfoam.tar.gz \ No newline at end of file From 8424e5778815a00aa2412f313acd2c3ac501ba6a Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 24 Jun 2026 22:34:10 +0530 Subject: [PATCH 04/19] Refine external test config shape per review on PR #732 Use external: instead of tutorials: for external suites, put source before path in examples, and move docs under Extending. --- tools/tests/README.md | 52 ++++---- tools/tests/systemtests/TestSuite.py | 183 ++++++++++++++++++--------- tools/tests/tests.yaml | 12 +- 3 files changed, 156 insertions(+), 91 deletions(-) diff --git a/tools/tests/README.md b/tools/tests/README.md index ee02c3b3e..f95a46bf1 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -126,6 +126,33 @@ Note that you will need to define the `TUTORIALS_REF` in the file [`reference_ve The results will be added to a Git LFS, but you will need special push access: just use the aforementioned GitHub Actions workflow, instead. +#### External test sources + +Local tutorials use the `tutorials:` list. For tutorials maintained in other git repositories or archives, use an `external:` list instead (a test suite defines one or the other, not both): + +```yaml +test_suites: + external: + external: + - source: + type: git + url: https://github.com/vidulejs/tutorials.git + ref: partitioned-heat-conduction-mixed-robin + subdir: . + path: partitioned-heat-conduction-robin + case_combination: + - left-openfoam + - right-openfoam + reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz +``` + +Supported `source.type` values: + +- `git`: shallow-clone `url` at `ref` (cached under `~/.cache/precice-tutorials` or `PRECICE_EXTERNAL_CACHE_DIR`). +- `archive`: download and extract a `.tar.gz` / `.zip` from `url`. + +The runner fetches the tutorial, copies it into the run directory, then continues with the usual Docker build/run and fieldcompare steps. `TUTORIALS_REF` / `TUTORIALS_PR` build arguments apply only to **local** tutorials; external tutorials are pinned by `source.ref`. Reference result paths are resolved relative to the fetched tutorial root. + ### Adding new components To add a new component, a few changes are needed: @@ -327,31 +354,6 @@ This template defines: - `volumes`: Directories mapped between the host and the container. Apart from directories relating to the users and groups, this also defines where to run the cases. - `command`: How to run a case depending on this component, including how and where to redirect any screen output. -#### External tutorial sources - -By default, every `path` must exist in the local `precice/tutorials` checkout. For tutorials maintained elsewhere, add an optional `source` block to the `tests.yaml` entry: - -```yaml -- path: flow-over-heated-plate - source: - type: git - url: https://github.com/precice/tutorials.git - ref: develop - subdir: . # optional subdirectory inside the repository - case_combination: - - fluid-openfoam - - solid-openfoam - reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz -``` - -Supported `type` values: - -- `local` (default): use the tutorial from this repository. -- `git`: shallow-clone `url` at `ref` (cached under `~/.cache/precice-tutorials` or `PRECICE_EXTERNAL_CACHE_DIR`). -- `archive`: download and extract a `.tar.gz` / `.zip` from `url`. - -The runner copies the resolved tutorial into the run directory, then continues with the usual Docker build/run and fieldcompare steps. `TUTORIALS_REF` / `TUTORIALS_PR` build arguments still apply only to **local** tutorials; external tutorials are pinned by `source.ref`. - ### Timeouts A `GLOBAL_TIMEOUT` is used for all operations. Its default value is 600s (5min), it is set in the beginning of [`Systemtests.py`](https://github.com/precice/tutorials/blob/develop/tools/tests/systemtests/Systemtest.py), and it can be overridden via the `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable. diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 1975e72c2..c739b4a6d 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -69,74 +69,137 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_times_of_tutorial = {} max_time_windows_of_tutorial = {} timeouts_of_tutorial = {} - for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: - source = TutorialSource.from_dict(tutorial_case.get('source')) - tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) - if not tutorial and source.type != "local": - tutorial_root = resolve_tutorial_root( - Path(tutorial_case['path']), - source, - PRECICE_EXTERNAL_CACHE_DIR, + suite_def = test_suites_raw[test_suite_name] + local_cases = suite_def.get('tutorials', []) + external_cases = suite_def.get('external', []) + if local_cases and external_cases: + raise ValueError( + f"Test suite '{test_suite_name}' must use either 'tutorials' or " + f"'external', not both." + ) + if not local_cases and not external_cases: + raise ValueError( + f"Test suite '{test_suite_name}' must define 'tutorials' or 'external'." + ) + + for tutorial_case in local_cases: + cls._add_tutorial_case( + tutorial_case, + TutorialSource.local(), + parsed_tutorials, + available_components, + case_combinations_of_tutorial, + reference_results_of_tutorial, + max_times_of_tutorial, + max_time_windows_of_tutorial, + timeouts_of_tutorial, + ) + + for tutorial_case in external_cases: + source_raw = tutorial_case.get('source') + if not source_raw: + raise ValueError( + f"External test entry in suite '{test_suite_name}' " + f"requires a 'source' block." ) - metadata_path = tutorial_root / "metadata.yaml" - if not metadata_path.exists(): - raise FileNotFoundError( - f"No metadata.yaml found for external tutorial " - f"{tutorial_case['path']} at {tutorial_root}" - ) - tutorial = Tutorial.from_yaml( - metadata_path, - available_components, - base_dir=tutorial_root.parent, - source=source, + source = TutorialSource.from_dict(source_raw) + if source.type == "local": + raise ValueError( + f"External test entry in suite '{test_suite_name}' " + f"must use a git or archive source." ) - parsed_tutorials.tutorials.append(tutorial) - if not tutorial: - raise Exception(f"No tutorial with path {tutorial_case['path']} found.") - if tutorial not in case_combinations_of_tutorial: - case_combinations_of_tutorial[tutorial] = [] - reference_results_of_tutorial[tutorial] = [] - max_times_of_tutorial[tutorial] = [] - max_time_windows_of_tutorial[tutorial] = [] - timeouts_of_tutorial[tutorial] = [] - - all_case_combinations = tutorial.case_combinations - case_combination_requested = CaseCombination.from_string_list( - tutorial_case['case_combination'], tutorial) - if case_combination_requested in all_case_combinations: - case_combinations_of_tutorial[tutorial].append(case_combination_requested) - ref_base = tutorial.path.parent if source.type != "local" else None - reference_results_of_tutorial[tutorial].append(ReferenceResult( - tutorial_case['reference_result'], - case_combination_requested, - base_dir=ref_base, - )) - max_time_raw = tutorial_case.get('max_time', None) - if max_time_raw is not None and (not isinstance( - max_time_raw, (int, float)) or max_time_raw <= 0): - raise ValueError(f"max_time must be a positive number, got {max_time_raw!r}") - max_times_of_tutorial[tutorial].append(max_time_raw) - mtw_raw = tutorial_case.get('max_time_windows', None) - if mtw_raw is not None and (not isinstance(mtw_raw, int) or mtw_raw <= 0): - raise ValueError(f"max_time_windows must be a positive integer, got {mtw_raw!r}") - max_time_windows_of_tutorial[tutorial].append(mtw_raw) - - timeout_value = tutorial_case.get('timeout', None) - if timeout_value is not None and not isinstance(timeout_value, int): - raise TypeError( - f"Expected 'timeout' to be an integer or None, but got {type(timeout_value).__name__} " - f"(value: {timeout_value}) in tutorial '{tutorial}'." - ) - timeouts_of_tutorial[tutorial].append(timeout_value) - else: - raise Exception( - f"Could not find the case combination {tutorial_case['case_combination']} in the current metadata of tutorial {tutorial.name}, or it does not define all necessary participants.") + cls._add_tutorial_case( + tutorial_case, + source, + parsed_tutorials, + available_components, + case_combinations_of_tutorial, + reference_results_of_tutorial, + max_times_of_tutorial, + max_time_windows_of_tutorial, + timeouts_of_tutorial, + ) testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial)) return cls(testsuites) + @staticmethod + def _add_tutorial_case( + tutorial_case: dict, + source: TutorialSource, + parsed_tutorials: Tutorials, + available_components: Components, + case_combinations_of_tutorial: Dict[Tutorial, List[CaseCombination]], + reference_results_of_tutorial: Dict[Tutorial, List[ReferenceResult]], + max_times_of_tutorial: Dict[Tutorial, list], + max_time_windows_of_tutorial: Dict[Tutorial, list], + timeouts_of_tutorial: Dict[Tutorial, List], + ) -> None: + tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) + if not tutorial and source.type != "local": + tutorial_root = resolve_tutorial_root( + Path(tutorial_case['path']), + source, + PRECICE_EXTERNAL_CACHE_DIR, + ) + metadata_path = tutorial_root / "metadata.yaml" + if not metadata_path.exists(): + raise FileNotFoundError( + f"No metadata.yaml found for external tutorial " + f"{tutorial_case['path']} at {tutorial_root}" + ) + tutorial = Tutorial.from_yaml( + metadata_path, + available_components, + base_dir=tutorial_root.parent, + source=source, + ) + parsed_tutorials.tutorials.append(tutorial) + if not tutorial: + raise Exception(f"No tutorial with path {tutorial_case['path']} found.") + if tutorial not in case_combinations_of_tutorial: + case_combinations_of_tutorial[tutorial] = [] + reference_results_of_tutorial[tutorial] = [] + max_times_of_tutorial[tutorial] = [] + max_time_windows_of_tutorial[tutorial] = [] + timeouts_of_tutorial[tutorial] = [] + + all_case_combinations = tutorial.case_combinations + case_combination_requested = CaseCombination.from_string_list( + tutorial_case['case_combination'], tutorial) + if case_combination_requested in all_case_combinations: + case_combinations_of_tutorial[tutorial].append(case_combination_requested) + ref_base = tutorial.path.parent if source.type != "local" else None + reference_results_of_tutorial[tutorial].append(ReferenceResult( + tutorial_case['reference_result'], + case_combination_requested, + base_dir=ref_base, + )) + max_time_raw = tutorial_case.get('max_time', None) + if max_time_raw is not None and (not isinstance( + max_time_raw, (int, float)) or max_time_raw <= 0): + raise ValueError(f"max_time must be a positive number, got {max_time_raw!r}") + max_times_of_tutorial[tutorial].append(max_time_raw) + mtw_raw = tutorial_case.get('max_time_windows', None) + if mtw_raw is not None and (not isinstance(mtw_raw, int) or mtw_raw <= 0): + raise ValueError(f"max_time_windows must be a positive integer, got {mtw_raw!r}") + max_time_windows_of_tutorial[tutorial].append(mtw_raw) + + timeout_value = tutorial_case.get('timeout', None) + if timeout_value is not None and not isinstance(timeout_value, int): + raise TypeError( + f"Expected 'timeout' to be an integer or None, but got {type(timeout_value).__name__} " + f"(value: {timeout_value}) in tutorial '{tutorial}'." + ) + timeouts_of_tutorial[tutorial].append(timeout_value) + else: + raise Exception( + f"Could not find the case combination {tutorial_case['case_combination']} " + f"in the current metadata of tutorial {tutorial.name}, or it does not " + f"define all necessary participants.") + def __iter__(self): return iter(self.testsuites) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 853b968cd..3ab49effc 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -765,24 +765,24 @@ test_suites: - *elastic-tube-1d_fluid-python_solid-python external: - tutorials: - - path: partitioned-heat-conduction-robin - source: + external: + - source: type: git url: https://github.com/vidulejs/tutorials.git ref: partitioned-heat-conduction-mixed-robin subdir: . + path: partitioned-heat-conduction-robin case_combination: - left-openfoam - right-openfoam reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz - - path: partitioned-heat-conduction-mixedbc - source: + - source: type: git url: https://github.com/vidulejs/tutorials.git ref: partitioned-heat-conduction-mixed-robin subdir: . + path: partitioned-heat-conduction-mixedbc case_combination: - left-openfoam - right-openfoam - reference_result: ./partitioned-heat-conduction-mixedbc/reference_results/left-openfoam_right-openfoam.tar.gz \ No newline at end of file + reference_result: ./partitioned-heat-conduction-mixedbc/reference_results/left-openfoam_right-openfoam.tar.gz \ No newline at end of file From 4b30bce3da838a9fa1f4b22c087b07f72a32ee1a Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 07:08:51 +0200 Subject: [PATCH 05/19] Revert change in Quickstart --- quickstart/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quickstart/README.md b/quickstart/README.md index 1d4d418de..3c83b7a2e 100644 --- a/quickstart/README.md +++ b/quickstart/README.md @@ -1,6 +1,8 @@ --- title: Quickstart permalink: quickstart.html +aliases: + - /quickstart.html keywords: tutorial, quickstart summary: "Install preCICE on Linux (e.g. via a Debian package) and couple an OpenFOAM fluid solver (using the OpenFOAM-preCICE adapter) with an example rigid body solver in C++." layout: "page" From 026816c5ee2299e2f6b9a7a60f2390f49100f0ee Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 07:17:03 +0200 Subject: [PATCH 06/19] Rearrange test suites --- tools/tests/tests.yaml | 63 +++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 98a4b95ed..a4233a6e2 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -565,6 +565,36 @@ test_suites: max_time: 20 reference_result: ./wolf-sheep-soil-creep/reference-results/soil-creep-landlab_wolf-sheep-grass-mesa.tar.gz +##################################################################### +## Externally-hosted test cases + partitioned-heat-conduction-robin: + external: + - &partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam + source: + type: git + url: https://github.com/vidulejs/tutorials.git + ref: partitioned-heat-conduction-mixed-robin + subdir: . + path: partitioned-heat-conduction-robin + case_combination: + - left-openfoam + - right-openfoam + skip_compare: true + reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz + - &partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam + source: + type: git + url: https://github.com/vidulejs/tutorials.git + ref: partitioned-heat-conduction-mixed-robin + subdir: . + path: partitioned-heat-conduction-mixedbc + case_combination: + - left-openfoam + - right-openfoam + skip_compare: true + reference_result: ./partitioned-heat-conduction-mixedbc/reference_results/left-openfoam_right-openfoam.tar.gz + + ##################################################################### ## Test suites referring to the test suites defined above @@ -628,6 +658,9 @@ test_suites: - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa + external: + - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam + - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam # These tests take longer to run. They are available, but not regularly executed. extra: @@ -639,6 +672,11 @@ test_suites: - *turek-hron-fsi3_fluid-nutils_solid-nutils - *two-scale-heat-conduction_macro-nutils_micro-nutils + external: + external: + - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam + - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam + # A selection of tests that cover a wide range of main features, meant for quicker CI executions precice: tutorials: @@ -770,28 +808,3 @@ test_suites: selected: tutorials: - *elastic-tube-1d_fluid-python_solid-python - - external: - external: - - source: - type: git - url: https://github.com/vidulejs/tutorials.git - ref: partitioned-heat-conduction-mixed-robin - subdir: . - path: partitioned-heat-conduction-robin - case_combination: - - left-openfoam - - right-openfoam - skip_compare: true - reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz - - source: - type: git - url: https://github.com/vidulejs/tutorials.git - ref: partitioned-heat-conduction-mixed-robin - subdir: . - path: partitioned-heat-conduction-mixedbc - case_combination: - - left-openfoam - - right-openfoam - skip_compare: true - reference_result: ./partitioned-heat-conduction-mixedbc/reference_results/left-openfoam_right-openfoam.tar.gz \ No newline at end of file From 87803f65351173172587183fc3168017b191764c Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 07:19:14 +0200 Subject: [PATCH 07/19] Convert a test case from git to zip --- tools/tests/tests.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index a4233a6e2..ad10046e4 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -583,10 +583,9 @@ test_suites: reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz - &partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam source: - type: git - url: https://github.com/vidulejs/tutorials.git - ref: partitioned-heat-conduction-mixed-robin - subdir: . + type: zip + url: https://github.com/vidulejs/tutorials/archive/refs/heads/partitioned-heat-conduction-mixed-robin.zip + subdir: tutorials-partitioned-heat-conduction-mixed-robin path: partitioned-heat-conduction-mixedbc case_combination: - left-openfoam From a07e6cb3134166173b5ccaa9c68d9b7259e248de Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 07:24:09 +0200 Subject: [PATCH 08/19] Fix type of archive --- tools/tests/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index ad10046e4..a06a6596d 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -583,7 +583,7 @@ test_suites: reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz - &partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam source: - type: zip + type: archive url: https://github.com/vidulejs/tutorials/archive/refs/heads/partitioned-heat-conduction-mixed-robin.zip subdir: tutorials-partitioned-heat-conduction-mixed-robin path: partitioned-heat-conduction-mixedbc From bd711a6e6f41117f51dadbdf2f1fa54fcc696fa2 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 07:30:45 +0200 Subject: [PATCH 09/19] Remove external from release --- tools/tests/tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index a06a6596d..e6733f5e0 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -657,9 +657,9 @@ test_suites: - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa - external: - - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam - - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam + # external: + # - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam + # - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam # These tests take longer to run. They are available, but not regularly executed. extra: From c9b1b6ec1e4b5cc015488467f7fe50a38a07a160 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 07:44:19 +0200 Subject: [PATCH 10/19] Set a longer timeous --- tools/tests/tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index e6733f5e0..07ab82e27 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -579,6 +579,7 @@ test_suites: case_combination: - left-openfoam - right-openfoam + timeout: 300 skip_compare: true reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz - &partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam @@ -590,6 +591,7 @@ test_suites: case_combination: - left-openfoam - right-openfoam + timeout: 300 skip_compare: true reference_result: ./partitioned-heat-conduction-mixedbc/reference_results/left-openfoam_right-openfoam.tar.gz From 1a46abdf66a1626fed0bbcdbb69a0d6c044785e8 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 08:06:06 +0200 Subject: [PATCH 11/19] Set longer timeous --- tools/tests/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 07ab82e27..dd572d834 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -579,7 +579,7 @@ test_suites: case_combination: - left-openfoam - right-openfoam - timeout: 300 + timeout: 480 skip_compare: true reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz - &partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam @@ -591,7 +591,7 @@ test_suites: case_combination: - left-openfoam - right-openfoam - timeout: 300 + timeout: 480 skip_compare: true reference_result: ./partitioned-heat-conduction-mixedbc/reference_results/left-openfoam_right-openfoam.tar.gz From a81795e0693d6753ce7c6c72e55867dec3cad743 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 08:27:59 +0200 Subject: [PATCH 12/19] Split test suite --- tools/tests/tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index dd572d834..97bbcdf41 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -582,6 +582,9 @@ test_suites: timeout: 480 skip_compare: true reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz + + partitioned-heat-conduction-mixedbc: + external: - &partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam source: type: archive From 0b29acb46154075284ae685cff7626448e53ea5f Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Wed, 1 Jul 2026 08:35:16 +0200 Subject: [PATCH 13/19] Improve documentation --- tools/tests/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/tests/README.md b/tools/tests/README.md index 4449bb2d0..b1245404e 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -172,22 +172,22 @@ The results will be added to a Git LFS, but you will need special push access: j #### External test sources -Local tutorials use the `tutorials:` list. For tutorials maintained in other git repositories or archives, use an `external:` list instead (a test suite defines one or the other, not both): +Local test cases are defined in the `tutorials:` list. For test cases maintained in other git repositories or available as archives, use the `external:` list instead (a test suite defines one or the other, not both): ```yaml test_suites: - external: + some-external-test-case: external: - source: type: git - url: https://github.com/vidulejs/tutorials.git - ref: partitioned-heat-conduction-mixed-robin + url: https://github.com/some-user/tutorials.git + ref: some-branch subdir: . - path: partitioned-heat-conduction-robin + path: path-to-test-case case_combination: - left-openfoam - right-openfoam - reference_result: ./partitioned-heat-conduction-robin/reference_results/left-openfoam_right-openfoam.tar.gz + reference_result: ./path-to-test-case/reference_results/participant-combination.tar.gz ``` Supported `source.type` values: @@ -195,7 +195,7 @@ Supported `source.type` values: - `git`: shallow-clone `url` at `ref` (cached under `~/.cache/precice-tutorials` or `PRECICE_EXTERNAL_CACHE_DIR`). - `archive`: download and extract a `.tar.gz` / `.zip` from `url`. -The runner fetches the tutorial, copies it into the run directory, then continues with the usual Docker build/run and fieldcompare steps. `TUTORIALS_REF` / `TUTORIALS_PR` build arguments apply only to **local** tutorials; external tutorials are pinned by `source.ref`. Reference result paths are resolved relative to the fetched tutorial root. +The runner fetches the tutorial, copies it into the run directory, and then continues with the usual Docker build/run and fieldcompare steps. `TUTORIALS_REF` / `TUTORIALS_PR` build arguments apply only to test cases sourced from the tutorials repository; external test cases are pinned by `source.ref`. Reference result paths are resolved relative to the root directory of the fetched test case. ### Adding new components From b1f07e420bdd6418090b172698413a7ba9227bc6 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 1 Jul 2026 21:28:49 +0530 Subject: [PATCH 14/19] Address Jul 1 review on external test sources Allow mixed tutorials/external suites, fail fast on bad git refs, restore shell execute bits after ZIP extract, align fetch timeouts with GLOBAL_TIMEOUT, and document external cache behaviour in README. --- tools/tests/README.md | 15 ++++-- tools/tests/metadata_parser/metdata.py | 3 +- tools/tests/systemtests/TestSuite.py | 5 -- tools/tests/systemtests/sources.py | 71 +++++++++----------------- tools/tests/tests.yaml | 6 +-- 5 files changed, 40 insertions(+), 60 deletions(-) diff --git a/tools/tests/README.md b/tools/tests/README.md index b1245404e..817e94ff0 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -172,13 +172,14 @@ The results will be added to a Git LFS, but you will need special push access: j #### External test sources -Local test cases are defined in the `tutorials:` list. For test cases maintained in other git repositories or available as archives, use the `external:` list instead (a test suite defines one or the other, not both): +Local test cases are defined in the `tutorials:` list. For test cases maintained in other git repositories or available as archives, use the `external:` list. A test suite may define both lists so that external cases can be referenced via YAML anchors alongside local tutorials (for example in the `release` suite). ```yaml test_suites: some-external-test-case: external: - - source: + - &some-external-test-case_left-openfoam_right-openfoam + source: type: git url: https://github.com/some-user/tutorials.git ref: some-branch @@ -188,13 +189,21 @@ test_suites: - left-openfoam - right-openfoam reference_result: ./path-to-test-case/reference_results/participant-combination.tar.gz + + release: + tutorials: + - *quickstart_openfoam_cpp + external: + - *some-external-test-case_left-openfoam_right-openfoam ``` Supported `source.type` values: -- `git`: shallow-clone `url` at `ref` (cached under `~/.cache/precice-tutorials` or `PRECICE_EXTERNAL_CACHE_DIR`). +- `git`: shallow-clone `url` at `ref`. The `ref` must exist on the remote; clone failures are reported as errors. - `archive`: download and extract a `.tar.gz` / `.zip` from `url`. +Fetched external sources are cached under `~/.cache/precice-tutorials` (or `PRECICE_EXTERNAL_CACHE_DIR`) so that repeated test runs and CI matrix jobs do not re-download the same repository or archive every time. The cache key is derived from the source URL, ref, and optional subdir. + The runner fetches the tutorial, copies it into the run directory, and then continues with the usual Docker build/run and fieldcompare steps. `TUTORIALS_REF` / `TUTORIALS_PR` build arguments apply only to test cases sourced from the tutorials repository; external test cases are pinned by `source.ref`. Reference result paths are resolved relative to the root directory of the fetched test case. ### Adding new components diff --git a/tools/tests/metadata_parser/metdata.py b/tools/tests/metadata_parser/metdata.py index dde2f1bc3..1f14b5dc3 100644 --- a/tools/tests/metadata_parser/metdata.py +++ b/tools/tests/metadata_parser/metdata.py @@ -6,7 +6,6 @@ import itertools from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR -# Import TutorialSource from systemtests.sources (used for external tutorial sources). from systemtests.sources import TutorialSource @@ -286,7 +285,7 @@ def from_cases_tuple(cls, cases: Tuple[Case], tutorial: Tutorial): class ReferenceResult: path: Path case_combination: CaseCombination - base_dir: Optional[Path] = None + base_dir: Path | None = None def __repr__(self) -> str: return f"{self.path.as_posix()}" diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index f2f5ff70c..3f5d35a22 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -80,11 +80,6 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): suite_def = test_suites_raw[test_suite_name] local_cases = suite_def.get('tutorials', []) external_cases = suite_def.get('external', []) - if local_cases and external_cases: - raise ValueError( - f"Test suite '{test_suite_name}' must use either 'tutorials' or " - f"'external', not both." - ) if not local_cases and not external_cases: raise ValueError( f"Test suite '{test_suite_name}' must define 'tutorials' or 'external'." diff --git a/tools/tests/systemtests/sources.py b/tools/tests/systemtests/sources.py index 7f4e6ad92..6b3224f94 100644 --- a/tools/tests/systemtests/sources.py +++ b/tools/tests/systemtests/sources.py @@ -6,12 +6,15 @@ import logging import os import shutil +import stat import subprocess import tarfile import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Optional + +# Same env var as Systemtest.GLOBAL_TIMEOUT (not imported to avoid circular deps). +_FETCH_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 180)) # Cache directory for fetched tutorials. Can be overridden via PRECICE_EXTERNAL_CACHE_DIR env. _DEFAULT_CACHE = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "precice-tutorials" @@ -20,12 +23,12 @@ @dataclass class TutorialSource: - """Describes where a tutorial is sourced from.""" + """Describes where a test case (tutorial) is sourced from (tutorials repository or external source).""" type: str # "local" | "git" | "archive" - url: Optional[str] = None - ref: Optional[str] = None - subdir: Optional[str] = None + url: str | None = None + ref: str | None = None + subdir: str | None = None @classmethod def local(cls) -> "TutorialSource": @@ -43,7 +46,7 @@ def from_dict(cls, data: dict) -> "TutorialSource": ) -def _cache_key(prefix: str, url: str, ref: Optional[str] = None, subdir: Optional[str] = None) -> str: +def _cache_key(prefix: str, url: str, ref: str | None = None, subdir: str | None = None) -> str: """Generate a short content-addressable cache key.""" parts = [url] if ref: @@ -54,7 +57,13 @@ def _cache_key(prefix: str, url: str, ref: Optional[str] = None, subdir: Optiona return hashlib.sha256(raw.encode()).hexdigest()[:16] -def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: Optional[str] = None) -> Path: +def _restore_shell_script_permissions(root: Path) -> None: + """Restore execute bits on shell scripts (zip extraction strips them).""" + for script in root.rglob("*.sh"): + script.chmod(script.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: str | None = None) -> Path: """ Clone or update a git repository and return the path to the checkout. If subdir is given, returns the path to that subdirectory within the repo. @@ -69,13 +78,13 @@ def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: Optional[str] = ["git", "-C", str(checkout), "fetch", "origin", ref, "--depth", "1"], check=True, capture_output=True, - timeout=120, + timeout=_FETCH_TIMEOUT, ) subprocess.run( ["git", "-C", str(checkout), "checkout", "FETCH_HEAD"], check=True, capture_output=True, - timeout=60, + timeout=_FETCH_TIMEOUT, ) except subprocess.CalledProcessError as e: logging.warning(f"Git fetch/checkout failed for {url}, recloning: {e}") @@ -86,47 +95,13 @@ def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: Optional[str] = ["git", "clone", "--depth", "1", "--branch", ref, url, str(checkout)], capture_output=True, text=True, - timeout=300, + timeout=_FETCH_TIMEOUT, ) if result.returncode != 0: - # Fallback: branch may not exist (e.g. repo uses develop/master instead of main) - # Clone without branch, then fetch and checkout ref (with common aliases) shutil.rmtree(checkout, ignore_errors=True) - logging.debug( - f"git clone --branch {ref} failed ({result.stderr}), trying clone + fetch" - ) - subprocess.run( - ["git", "clone", "--depth", "1", url, str(checkout)], - check=True, - capture_output=True, - timeout=300, + raise RuntimeError( + f"git clone --branch {ref!r} failed for {url}: {result.stderr}" ) - refs_to_try = [ref] - if ref == "main": - refs_to_try.extend(["develop", "master"]) - elif ref == "master": - refs_to_try.extend(["main", "develop"]) - last_err = None - for r in refs_to_try: - res = subprocess.run( - ["git", "-C", str(checkout), "fetch", "origin", r, "--depth", "1"], - capture_output=True, - text=True, - timeout=120, - ) - if res.returncode == 0: - subprocess.run( - ["git", "-C", str(checkout), "checkout", "FETCH_HEAD"], - check=True, - capture_output=True, - timeout=60, - ) - break - last_err = res.stderr - else: - raise RuntimeError( - f"Could not fetch ref '{ref}' (tried {refs_to_try}): {last_err}" - ) if subdir: subpath = checkout / subdir @@ -136,7 +111,7 @@ def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: Optional[str] = return checkout -def fetch_archive(url: str, cache_dir: Path, subdir: Optional[str] = None) -> Path: +def fetch_archive(url: str, cache_dir: Path, subdir: str | None = None) -> Path: """ Download and extract an archive (tar.gz, tar, zip) and return the path. """ @@ -147,6 +122,7 @@ def fetch_archive(url: str, cache_dir: Path, subdir: Optional[str] = None) -> Pa extract_dir = cache_dir / key if extract_dir.exists(): + _restore_shell_script_permissions(extract_dir) return extract_dir / subdir if subdir else extract_dir with tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") as tmp: @@ -164,6 +140,7 @@ def fetch_archive(url: str, cache_dir: Path, subdir: Optional[str] = None) -> Pa with zipfile.ZipFile(tmp_path, "r") as zf: zf.extractall(extract_dir) + _restore_shell_script_permissions(extract_dir) finally: tmp_path.unlink(missing_ok=True) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 97bbcdf41..7ec189189 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -662,9 +662,9 @@ test_suites: - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa - # external: - # - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam - # - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam + external: + - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam + - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam # These tests take longer to run. They are available, but not regularly executed. extra: From 47dfcf77dfcda348af6d9e9cf7ee081e7a8da81c Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Fri, 3 Jul 2026 07:02:35 +0530 Subject: [PATCH 15/19] Clarify shell-script executable-bit helper Rename to _ensure_shell_scripts_executable and document that it adds execute bits to every .sh in extracted archives (originals unknown), rather than restoring original permissions. --- tools/tests/systemtests/sources.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/tests/systemtests/sources.py b/tools/tests/systemtests/sources.py index 6b3224f94..e3c3b9388 100644 --- a/tools/tests/systemtests/sources.py +++ b/tools/tests/systemtests/sources.py @@ -57,8 +57,14 @@ def _cache_key(prefix: str, url: str, ref: str | None = None, subdir: str | None return hashlib.sha256(raw.encode()).hexdigest()[:16] -def _restore_shell_script_permissions(root: Path) -> None: - """Restore execute bits on shell scripts (zip extraction strips them).""" +def _ensure_shell_scripts_executable(root: Path) -> None: + """Mark all .sh files under root as executable. + + ZIP extraction does not preserve Unix permission bits, so run.sh and other + tutorial scripts lose their execute bit. The original modes are not known, + so this unconditionally adds execute bits to every .sh file. Intended only + for extracted tutorial/test-case archives. + """ for script in root.rglob("*.sh"): script.chmod(script.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) @@ -122,7 +128,7 @@ def fetch_archive(url: str, cache_dir: Path, subdir: str | None = None) -> Path: extract_dir = cache_dir / key if extract_dir.exists(): - _restore_shell_script_permissions(extract_dir) + _ensure_shell_scripts_executable(extract_dir) return extract_dir / subdir if subdir else extract_dir with tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") as tmp: @@ -140,7 +146,7 @@ def fetch_archive(url: str, cache_dir: Path, subdir: str | None = None) -> Path: with zipfile.ZipFile(tmp_path, "r") as zf: zf.extractall(extract_dir) - _restore_shell_script_permissions(extract_dir) + _ensure_shell_scripts_executable(extract_dir) finally: tmp_path.unlink(missing_ok=True) From 10b3fbab1b3c44434f3b395a8e9e5ea63a4f3b2b Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Fri, 3 Jul 2026 10:15:13 +0200 Subject: [PATCH 16/19] Update/reorder list of tests --- tools/tests/tests.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 7ec189189..1d30d28bd 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -569,7 +569,7 @@ test_suites: ## Externally-hosted test cases partitioned-heat-conduction-robin: external: - - &partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam + - &partitioned-heat-conduction-robin_left-openfoam_right-openfoam source: type: git url: https://github.com/vidulejs/tutorials.git @@ -663,8 +663,8 @@ test_suites: - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa external: - - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam + - *partitioned-heat-conduction-robin_left-openfoam_right-openfoam # These tests take longer to run. They are available, but not regularly executed. extra: @@ -678,8 +678,8 @@ test_suites: external: external: - - *partitioned-heat-conduction-robin_lrft-openfoam_right-openfoam - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam + - *partitioned-heat-conduction-robin_left-openfoam_right-openfoam # A selection of tests that cover a wide range of main features, meant for quicker CI executions precice: @@ -798,6 +798,9 @@ test_suites: - *quickstart_openfoam_cpp - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam + external: + - *partitioned-heat-conduction-mixedbc_left-openfoam_right-openfoam + - *partitioned-heat-conduction-robin_left-openfoam_right-openfoam su2-adapter: tutorials: From 8b557342c05da1a9355d76da11cadeb53a4f82cb Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Fri, 3 Jul 2026 15:39:04 +0530 Subject: [PATCH 17/19] Log external fetch/cache and prefix External: in case names Add log messages when an external case is fetched vs. loaded from cache (git and archive), and prepend "External:" to the case name in logs, as requested. --- tools/tests/systemtests/Systemtest.py | 3 ++- tools/tests/systemtests/sources.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index c19b6ec3e..a453db64e 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -982,7 +982,8 @@ def _run_tutorial(self): return DockerComposeResult(exit_code, stdout_data, stderr_data, self, elapsed_time) def __repr__(self): - return f"{self.tutorial.name} {self.case_combination}" + prefix = "External: " if getattr(self.tutorial.source, "type", "local") != "local" else "" + return f"{prefix}{self.tutorial.name} {self.case_combination}" def __apply_max_time_override(self): """Overwrite or value in precice-config.xml.""" diff --git a/tools/tests/systemtests/sources.py b/tools/tests/systemtests/sources.py index e3c3b9388..ee9fce94c 100644 --- a/tools/tests/systemtests/sources.py +++ b/tools/tests/systemtests/sources.py @@ -79,6 +79,7 @@ def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: str | None = Non checkout = cache_dir / key if checkout.exists(): + logging.info(f"Using cached external git source {url} (ref {ref}) from {checkout}") try: subprocess.run( ["git", "-C", str(checkout), "fetch", "origin", ref, "--depth", "1"], @@ -97,6 +98,7 @@ def fetch_git_repo(url: str, ref: str, cache_dir: Path, subdir: str | None = Non shutil.rmtree(checkout, ignore_errors=True) if not checkout.exists(): + logging.info(f"Fetching external git source {url} (ref {ref}) into {checkout}") result = subprocess.run( ["git", "clone", "--depth", "1", "--branch", ref, url, str(checkout)], capture_output=True, @@ -128,13 +130,14 @@ def fetch_archive(url: str, cache_dir: Path, subdir: str | None = None) -> Path: extract_dir = cache_dir / key if extract_dir.exists(): + logging.info(f"Using cached external archive {url} from {extract_dir}") _ensure_shell_scripts_executable(extract_dir) return extract_dir / subdir if subdir else extract_dir with tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") as tmp: tmp_path = Path(tmp.name) try: - logging.info(f"Downloading {url}") + logging.info(f"Fetching external archive {url} into {extract_dir}") urllib.request.urlretrieve(url, tmp_path) extract_dir.mkdir(parents=True, exist_ok=True) From 32e81c593b24460a71d48ac0e219458037450f47 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Fri, 3 Jul 2026 18:33:21 +0530 Subject: [PATCH 18/19] Resolve external tutorials once at parse time Store resolved_root on Tutorial and reuse it when copying into the run directory to avoid duplicate fetch/cache logs and redundant git updates. --- tools/tests/metadata_parser/metdata.py | 3 +++ tools/tests/systemtests/Systemtest.py | 4 +++- tools/tests/systemtests/TestSuite.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/tests/metadata_parser/metdata.py b/tools/tests/metadata_parser/metdata.py index 1f14b5dc3..0375bf987 100644 --- a/tools/tests/metadata_parser/metdata.py +++ b/tools/tests/metadata_parser/metdata.py @@ -308,6 +308,9 @@ class Tutorial: participants: List[str] cases: List[Case] source: "TutorialSource" = field(default_factory=TutorialSource.local) + # Filesystem path to the fetched external tutorial, resolved once at parse + # time and reused when copying into the run directory (None for local). + resolved_root: "Path | None" = None case_combinations: List[CaseCombination] = field(init=False) def __post_init__(self): diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index a453db64e..021fe7255 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -500,7 +500,9 @@ def __copy_tutorial_into_directory(self, run_directory: Path): self.tutorial_folder = slugify( f'{self.tutorial.path.name}_{self.case_combination.cases}_{current_time_string}') destination = run_directory / self.tutorial_folder - src = resolve_tutorial_root( + # External sources are fetched and resolved once at parse time; reuse + # that path here to avoid a redundant fetch (and duplicate log line). + src = self.tutorial.resolved_root or resolve_tutorial_root( self.tutorial.path, self.tutorial.source, PRECICE_EXTERNAL_CACHE_DIR, diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 3f5d35a22..adf063988 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -181,6 +181,7 @@ def _add_tutorial_case( base_dir=tutorial_root.parent, source=source, ) + tutorial.resolved_root = tutorial_root parsed_tutorials.tutorials.append(tutorial) if not tutorial: raise Exception(f"No tutorial with path {tutorial_case['path']} found.") From 865edb6cfa55ee5ed7ceb6eaa925c18f080cb20c Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Fri, 3 Jul 2026 15:56:35 +0200 Subject: [PATCH 19/19] Apply suggestion from @MakisH --- tools/tests/README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/tests/README.md b/tools/tests/README.md index 817e94ff0..67d48cacb 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -178,7 +178,7 @@ Local test cases are defined in the `tutorials:` list. For test cases maintained test_suites: some-external-test-case: external: - - &some-external-test-case_left-openfoam_right-openfoam + - &some-external-test-case source: type: git url: https://github.com/some-user/tutorials.git @@ -186,15 +186,13 @@ test_suites: subdir: . path: path-to-test-case case_combination: - - left-openfoam - - right-openfoam - reference_result: ./path-to-test-case/reference_results/participant-combination.tar.gz + ... release: tutorials: - *quickstart_openfoam_cpp external: - - *some-external-test-case_left-openfoam_right-openfoam + - *some-external-test-case ``` Supported `source.type` values: