From d43708298bd378e4744822d6b78c8498b9fa861f Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 17 Jun 2026 14:08:33 +0530 Subject: [PATCH 1/8] Add per-test fieldcompare tolerance and skip_compare (#837) Allow overriding fieldcompare rtol via tests.yaml and optionally skip the compare step. Re-enable previously quarantined tests with tolerance 1e-2 for near-zero exported fields. --- changelog-entries/837.md | 1 + tools/tests/README.md | 6 ++- ...docker-compose.field_compare.template.yaml | 2 +- tools/tests/generate_reference_results.py | 8 ++- tools/tests/systemtests.py | 8 ++- tools/tests/systemtests/Systemtest.py | 49 +++++++++++++------ tools/tests/systemtests/TestSuite.py | 30 +++++++++++- tools/tests/tests.yaml | 30 +++++------- 8 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 changelog-entries/837.md diff --git a/changelog-entries/837.md b/changelog-entries/837.md new file mode 100644 index 000000000..6b8ba6e1d --- /dev/null +++ b/changelog-entries/837.md @@ -0,0 +1 @@ +- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration ([#837](https://github.com/precice/tutorials/issues/837)). diff --git a/tools/tests/README.md b/tools/tests/README.md index 620b76164..4d0d62088 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -25,7 +25,7 @@ The available test suites are found in [`tests.yaml`](https://github.com/precice - `precice` is a subset of cases that cover a range of preCICE features - `release` is for all available but some very long or known to fail tests - `extra` is for some longer tests -- `expected-to-fail`, `selected`, and `system-tests-dev` for some special cases +- `selected` and `system-tests-dev` for some special cases The `Use workflow from` is a default option of GitHub Actions that concerns the GHA workflow file itself. @@ -106,6 +106,8 @@ fieldcompare dir precice-exports/ reference-results-unpacked// \ -rtol 3e-7 ``` +The default relative tolerance is `3e-7`. Per-test overrides are available in `tests.yaml` via `tolerance` (passed to fieldcompare as `-rtol`). Set `skip_compare: true` to skip the comparison step and only verify that build and run succeed. + The differences are only shown per file, and there is no global metric or other summary (see [related discussion in fieldcompare](https://gitlab.com/dglaeser/fieldcompare/-/work_items/69)). Alternatively, [visualize the `precice-exports/diff_*.vtu` in ParaView](https://precice.org/configuration-export.html#visualization-with-paraview). @@ -333,4 +335,6 @@ A `GLOBAL_TIMEOUT` is used for all operations. Its default value is 600s (5min), Tests can define a different `timeout` in their `tests.yaml` entry, which applies to the running and results comparison steps. +Tests can define a different `tolerance` in their `tests.yaml` entry, which applies to the fieldcompare step (relative tolerance, `-rtol`). The default is `3e-7`. Use `skip_compare: true` to skip fieldcompare entirely. + diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 8f6dd5e19..65590ab61 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -6,4 +6,4 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} - - "-rtol 3e-7 --ignore-missing-reference-files --diff" + - "-rtol {{ tolerance }} --ignore-missing-reference-files --diff" diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index d7470c8a6..698d11c85 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -2,7 +2,7 @@ from metadata_parser.metdata import Tutorials, ReferenceResult from systemtests.TestSuite import TestSuites from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL from pathlib import Path from typing import List from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR @@ -145,13 +145,17 @@ def main(): max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) timeouts = test_suite.timeouts.get(tutorial, []) + tolerances = test_suite.tolerances.get(tutorial, []) + skip_compares = test_suite.skip_compares.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT + tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 3183b0ca6..90c6bac62 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -2,7 +2,7 @@ import argparse from pathlib import Path from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL, display_systemtestresults_as_table from systemtests.TestSuite import TestSuites from metadata_parser.metdata import Tutorials, Case import logging @@ -89,13 +89,17 @@ def _group_end() -> None: max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) timeouts = test_suite.timeouts.get(tutorial, []) + tolerances = test_suite.tolerances.get(tutorial, []) + skip_compares = test_suite.skip_compares.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT + tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 932b6b944..f2a6bea00 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -21,6 +21,7 @@ GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 600)) +DEFAULT_FIELDCOMPARE_RTOL = 3e-7 SHORT_TIMEOUT = 10 DIFF_RESULTS_DIR = "diff-results" @@ -219,6 +220,8 @@ class Systemtest: max_time: float | None = None max_time_windows: int | None = None timeout: int = GLOBAL_TIMEOUT + tolerance: float = DEFAULT_FIELDCOMPARE_RTOL + skip_compare: bool = False params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -330,6 +333,7 @@ def __get_field_compare_compose_file(self): 'tutorial_folder': self.tutorial_folder, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, 'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""), + 'tolerance': self.tolerance, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template( @@ -669,6 +673,15 @@ def _run_field_compare(self): elapsed_time = time.perf_counter() - time_start return FieldCompareResult(exit_code, stdout_data, stderr_data, self, elapsed_time) + def _log_skipped_fieldcompare(self) -> None: + log_sink = getattr(self, "_log_sink", None) + if log_sink is not None: + log_sink.begin_stage("compare") + log_sink.append_stdout( + f"(skipped: skip_compare=true, default rtol would be {DEFAULT_FIELDCOMPARE_RTOL})", + "compare", + ) + def __archive_fieldcompare_diffs(self) -> None: """ Copy fieldcompare diff VTK files from precice-exports/ into diff-results/, @@ -833,20 +846,26 @@ def run(self, run_directory: Path): solver_time=docker_run_result.runtime, fieldcompare_time=0) - fieldcompare_result = self._run_field_compare() - std_out.extend(fieldcompare_result.stdout_data) - std_err.extend(fieldcompare_result.stderr_data) - if fieldcompare_result.exit_code != 0: - self.__archive_fieldcompare_diffs() - logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") - return SystemtestResult( - False, - std_out, - std_err, - self, - build_time=docker_build_result.runtime, - solver_time=docker_run_result.runtime, - fieldcompare_time=fieldcompare_result.runtime) + if self.skip_compare: + logging.info(f"Skipping fieldcompare for {self} (skip_compare=true)") + self._log_skipped_fieldcompare() + fieldcompare_time = 0.0 + else: + fieldcompare_result = self._run_field_compare() + std_out.extend(fieldcompare_result.stdout_data) + std_err.extend(fieldcompare_result.stderr_data) + if fieldcompare_result.exit_code != 0: + self.__archive_fieldcompare_diffs() + logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=fieldcompare_result.runtime) + fieldcompare_time = fieldcompare_result.runtime # self.__cleanup() self._cleanup_docker_networks() @@ -857,7 +876,7 @@ def run(self, run_directory: Path): self, build_time=docker_build_result.runtime, solver_time=docker_run_result.runtime, - fieldcompare_time=fieldcompare_result.runtime) + fieldcompare_time=fieldcompare_time) def run_for_reference_results(self, run_directory: Path): """ diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index a498ca4bd..a561e9ddf 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -13,6 +13,8 @@ class TestSuite: max_times: Dict[Tutorial, list] = field(default_factory=dict) max_time_windows: Dict[Tutorial, list] = field(default_factory=dict) timeouts: Dict[Tutorial, List] = field(default_factory=dict) + tolerances: Dict[Tutorial, list] = field(default_factory=dict) + skip_compares: Dict[Tutorial, list] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -54,6 +56,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_times_of_tutorial = {} max_time_windows_of_tutorial = {} timeouts_of_tutorial = {} + tolerances_of_tutorial = {} + skip_compares_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -66,6 +70,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_times_of_tutorial[tutorial] = [] max_time_windows_of_tutorial[tutorial] = [] timeouts_of_tutorial[tutorial] = [] + tolerances_of_tutorial[tutorial] = [] + skip_compares_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -91,12 +97,34 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): f"(value: {timeout_value}) in tutorial '{tutorial}'." ) timeouts_of_tutorial[tutorial].append(timeout_value) + + tolerance_value = tutorial_case.get('tolerance', None) + if tolerance_value is not None: + if isinstance(tolerance_value, str): + try: + tolerance_value = float(tolerance_value) + except ValueError as exc: + raise ValueError( + f"tolerance must be a positive number, got {tolerance_value!r}") from exc + if not isinstance(tolerance_value, (int, float)) or tolerance_value <= 0: + raise ValueError( + f"tolerance must be a positive number, got {tolerance_value!r}") + tolerances_of_tutorial[tutorial].append(tolerance_value) + + skip_compare_value = tutorial_case.get('skip_compare', None) + if skip_compare_value is not None and not isinstance(skip_compare_value, bool): + raise TypeError( + f"Expected 'skip_compare' to be a boolean or None, but got " + f"{type(skip_compare_value).__name__} (value: {skip_compare_value}) " + f"in tutorial '{tutorial}'." + ) + skip_compares_of_tutorial[tutorial].append(skip_compare_value) else: raise Exception( f"Could not find the case combination {tutorial_case['case_combination']} in the current metadata of tutorial {tutorial.name}, or it does not define all necessary participants.") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial)) + reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial, tolerances_of_tutorial, skip_compares_of_tutorial)) return cls(testsuites) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 70faade1d..926193ded 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -114,7 +114,8 @@ test_suites: - fluid-openfoam - solid-fenics max_time_windows: 1 - reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz # Too small values, expected to fail the comparisons. + tolerance: 1e-2 + reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz flow-around-controlled-moving-cylinder: tutorials: @@ -487,14 +488,16 @@ test_suites: case_combination: - macro-dumux - micro-dumux - reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz # Too small values, expected to fail the comparisons. + tolerance: 1e-2 + reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz - &two-scale-heat-conduction_macro-nutils_micro-nutils path: two-scale-heat-conduction case_combination: - macro-nutils - micro-nutils max_time: 0.05 - reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz # Too small values, expected to fail the comparisons. + tolerance: 1e-2 + reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz volume-coupled-diffusion: tutorials: @@ -605,11 +608,6 @@ test_suites: - *perpendicular-flap_fluid-openfoam_solid-nutils - *turek-hron-fsi3_fluid-nutils_solid-nutils - expected-to-fail: - tutorials: - - *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare - - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare - # A selection of tests that cover a wide range of main features, meant for quicker CI executions precice: tutorials: @@ -653,9 +651,7 @@ test_suites: - *perpendicular-flap_fluid-openfoam_solid-fenics - *perpendicular-flap_fluid-su2_solid-fenics - *volume-coupled-diffusion_source-fenics_drain-fenics - - # Excluded: - # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare + - *elastic-tube-3d_fluid-openfoam_solid-fenics fenicsx-adapter: tutorials: @@ -678,8 +674,8 @@ test_suites: micro-manager: tutorials: - - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare - - *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare + - *two-scale-heat-conduction_macro-dumux_micro-dumux + - *two-scale-heat-conduction_macro-nutils_micro-nutils nutils-adapter: # Not a repository tutorials: @@ -694,9 +690,7 @@ test_suites: - *turek-hron-fsi3_fluid-nutils_solid-nutils - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - - # Excluded: - # *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare + - *two-scale-heat-conduction_macro-nutils_micro-nutils openfoam-adapter: tutorials: @@ -729,9 +723,7 @@ test_suites: - *quickstart_openfoam_cpp - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - - # Excluded: - # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare + - *elastic-tube-3d_fluid-openfoam_solid-fenics su2-adapter: tutorials: From 98b91f661382995a5b7a5aadb9d217d332bcff88 Mon Sep 17 00:00:00 2001 From: Pranjal Date: Wed, 17 Jun 2026 14:42:28 +0530 Subject: [PATCH 2/8] Update changelog entry for issue #837 --- changelog-entries/837.md | 1 - changelog-entries/847.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog-entries/837.md create mode 100644 changelog-entries/847.md diff --git a/changelog-entries/837.md b/changelog-entries/837.md deleted file mode 100644 index 6b8ba6e1d..000000000 --- a/changelog-entries/837.md +++ /dev/null @@ -1 +0,0 @@ -- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration ([#837](https://github.com/precice/tutorials/issues/837)). diff --git a/changelog-entries/847.md b/changelog-entries/847.md new file mode 100644 index 000000000..a0ac31f1a --- /dev/null +++ b/changelog-entries/847.md @@ -0,0 +1 @@ +- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration closes #837. From da376f110b767864eec67b0d2373bf893613ff38 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 17 Jun 2026 14:45:32 +0530 Subject: [PATCH 3/8] Fix autopep8 line length in system test runners Wrap long tolerance/skip_compare lines so check_style pre-commit passes. --- tools/tests/generate_reference_results.py | 3 ++- tools/tests/systemtests.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 698d11c85..49decc7a7 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -152,7 +152,8 @@ def main(): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT - tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + tolerance = tolerances[i] if i < len( + tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False systemtests_to_run.add( Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 90c6bac62..61c8273bc 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -96,8 +96,10 @@ def _group_end() -> None: max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT - tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL - skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False + tolerance = tolerances[i] if i < len( + tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + skip_compare = skip_compares[i] if i < len( + skip_compares) and skip_compares[i] is not None else False systemtests_to_run.append( Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) From d7383fa2e59c8890938a0b406012b615814ec028 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 24 Jun 2026 23:21:37 +0530 Subject: [PATCH 4/8] Address review comments on per-test fieldcompare tolerance (#847) Sort re-enabled tests alphabetically, apply README/changelog wording, drop compare settings from generate_reference_results, and simplify skip_compare logging. --- changelog-entries/847.md | 2 +- tools/tests/README.md | 2 +- tools/tests/generate_reference_results.py | 9 ++------- tools/tests/systemtests/Systemtest.py | 10 ---------- tools/tests/tests.yaml | 6 +++--- 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/changelog-entries/847.md b/changelog-entries/847.md index a0ac31f1a..1b9b516da 100644 --- a/changelog-entries/847.md +++ b/changelog-entries/847.md @@ -1 +1 @@ -- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration closes #837. +- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration [#847](https://github.com/precice/tutorials/pull/847) diff --git a/tools/tests/README.md b/tools/tests/README.md index 4d0d62088..2314bc14c 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -106,7 +106,7 @@ fieldcompare dir precice-exports/ reference-results-unpacked// \ -rtol 3e-7 ``` -The default relative tolerance is `3e-7`. Per-test overrides are available in `tests.yaml` via `tolerance` (passed to fieldcompare as `-rtol`). Set `skip_compare: true` to skip the comparison step and only verify that build and run succeed. +The default relative tolerance (`-rtol`) is `3e-7`. Per-test overrides are possible in `tests.yaml` (e.g., `tolerance: 1e-2`). Set `skip_compare: true` to skip the comparison step and only verify that the build and run steps succeed. The differences are only shown per file, and there is no global metric or other summary (see [related discussion in fieldcompare](https://gitlab.com/dglaeser/fieldcompare/-/work_items/69)). diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 49decc7a7..d7470c8a6 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -2,7 +2,7 @@ from metadata_parser.metdata import Tutorials, ReferenceResult from systemtests.TestSuite import TestSuites from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT from pathlib import Path from typing import List from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR @@ -145,18 +145,13 @@ def main(): max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) timeouts = test_suite.timeouts.get(tutorial, []) - tolerances = test_suite.tolerances.get(tutorial, []) - skip_compares = test_suite.skip_compares.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT - tolerance = tolerances[i] if i < len( - tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL - skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index f2a6bea00..348cb909e 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -673,15 +673,6 @@ def _run_field_compare(self): elapsed_time = time.perf_counter() - time_start return FieldCompareResult(exit_code, stdout_data, stderr_data, self, elapsed_time) - def _log_skipped_fieldcompare(self) -> None: - log_sink = getattr(self, "_log_sink", None) - if log_sink is not None: - log_sink.begin_stage("compare") - log_sink.append_stdout( - f"(skipped: skip_compare=true, default rtol would be {DEFAULT_FIELDCOMPARE_RTOL})", - "compare", - ) - def __archive_fieldcompare_diffs(self) -> None: """ Copy fieldcompare diff VTK files from precice-exports/ into diff-results/, @@ -848,7 +839,6 @@ def run(self, run_directory: Path): if self.skip_compare: logging.info(f"Skipping fieldcompare for {self} (skip_compare=true)") - self._log_skipped_fieldcompare() fieldcompare_time = 0.0 else: fieldcompare_result = self._run_field_compare() diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 926193ded..577cbd3ee 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -644,6 +644,7 @@ test_suites: fenics-adapter: tutorials: - *channel-transport-reaction_fluid-fenics_chemical-fenics + - *elastic-tube-3d_fluid-openfoam_solid-fenics - *flow-over-heated-plate_fluid-openfoam_solid-fenics - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics - *partitioned-heat-conduction-complex_dirichlet-fenics_neumann-fenics @@ -651,7 +652,6 @@ test_suites: - *perpendicular-flap_fluid-openfoam_solid-fenics - *perpendicular-flap_fluid-su2_solid-fenics - *volume-coupled-diffusion_source-fenics_drain-fenics - - *elastic-tube-3d_fluid-openfoam_solid-fenics fenicsx-adapter: tutorials: @@ -688,9 +688,9 @@ test_suites: - *partitioned-pipe-multiscale_fluid1d-left-nutils_fluid3d-right-openfoam - *perpendicular-flap_fluid-nutils_solid-calculix - *turek-hron-fsi3_fluid-nutils_solid-nutils + - *two-scale-heat-conduction_macro-nutils_micro-nutils - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - - *two-scale-heat-conduction_macro-nutils_micro-nutils openfoam-adapter: tutorials: @@ -698,6 +698,7 @@ test_suites: - *channel-transport_fluid-openfoam_transport-nutils - *channel-transport-particles_fluid-openfoam_particles-mercurydpm - *elastic-tube-3d_fluid-openfoam_solid-calculix + - *elastic-tube-3d_fluid-openfoam_solid-fenics - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python - *flow-over-heated-plate_fluid-openfoam_solid-fenics - *flow-over-heated-plate_fluid-openfoam_solid-nutils @@ -723,7 +724,6 @@ test_suites: - *quickstart_openfoam_cpp - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - - *elastic-tube-3d_fluid-openfoam_solid-fenics su2-adapter: tutorials: From 9490cd9f86df303c83cd110da79172dd8b9bfc06 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Mon, 29 Jun 2026 18:54:06 +0200 Subject: [PATCH 5/8] Add elastic-tube-3d_fluid-openfoam_solid-fenics to release, disable comparison --- tools/tests/tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index c528c00e1..8bb90192b 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -121,7 +121,8 @@ test_suites: - fluid-openfoam - solid-fenics max_time_windows: 1 - tolerance: 1e-2 + tolerance: 1e-2 # Some values are too close to zero + skip_compare: true # Comparison takes too reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz flow-around-controlled-moving-cylinder: @@ -579,6 +580,7 @@ test_suites: - *elastic-tube-1d_fluid-fortran-module_solid-fortran-module - *elastic-tube-1d_fluid-python_solid-python - *elastic-tube-3d_fluid-openfoam_solid-calculix + - *elastic-tube-3d_fluid-openfoam_solid-fenics - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python - *flow-over-heated-plate_fluid-openfoam_solid-fenics - *flow-over-heated-plate_fluid-openfoam_solid-fenicsx From e8336eb3bafa82544db497de6e9ff84fdd1a89a0 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Mon, 29 Jun 2026 19:19:51 +0200 Subject: [PATCH 6/8] Disable comparisons for two-scale-heat-conduction --- tools/tests/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 8bb90192b..1e55d3b6c 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -511,7 +511,7 @@ test_suites: case_combination: - macro-dumux - micro-dumux - tolerance: 1e-2 + skip_compare: true # Values too close to zero reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz - &two-scale-heat-conduction_macro-nutils_micro-nutils path: two-scale-heat-conduction @@ -519,7 +519,7 @@ test_suites: - macro-nutils - micro-nutils max_time: 0.05 - tolerance: 1e-2 + skip_compare: true # Values too close to zero reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz volume-coupled-diffusion: From 3733b605b61a3a424d59b654cc33096f8f5a2676 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Mon, 29 Jun 2026 19:40:32 +0200 Subject: [PATCH 7/8] Explicitly run micro cases in serial --- two-scale-heat-conduction/metadata.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/two-scale-heat-conduction/metadata.yaml b/two-scale-heat-conduction/metadata.yaml index 6c7c1dafb..bb0f361cd 100644 --- a/two-scale-heat-conduction/metadata.yaml +++ b/two-scale-heat-conduction/metadata.yaml @@ -16,17 +16,17 @@ cases: macro-nutils: participant: Macro directory: ./macro-nutils - run: ./run.sh -l /home/precice/nutils + run: ./run.sh component: nutils-adapter micro-dumux: participant: Micro-Manager directory: ./micro-dumux - run: source /home/precice/venv/bin/activate && ./run.sh -l /home/precice/dumux + run: source /home/precice/venv/bin/activate && ./run.sh -s -l /home/precice/dumux component: dumux-adapter micro-nutils: participant: Micro-Manager directory: ./micro-nutils - run: source /home/precice/venv/bin/activate && ./run.sh -l /home/precice/nutils + run: source /home/precice/venv/bin/activate && ./run.sh -s component: nutils-adapter \ No newline at end of file From decbc6376aab4edd5eba44145d091bcbc2c5e980 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Mon, 29 Jun 2026 19:52:06 +0200 Subject: [PATCH 8/8] Add two-scale-heat-conduction_macro-dumux_micro-dumux to release test suite --- tools/tests/tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 1e55d3b6c..4901ecfa0 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -621,6 +621,7 @@ test_suites: - *quickstart_openfoam_cpp - *resonant-circuit_capacitor-python_coil-python - *turek-hron-fsi3_fluid-openfoam_solid-dealii + - *two-scale-heat-conduction_macro-dumux_micro-dumux - *volume-coupled-diffusion_source-fenics_drain-fenics - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam @@ -634,6 +635,7 @@ test_suites: - *perpendicular-flap_fluid-nutils_solid-calculix - *perpendicular-flap_fluid-openfoam_solid-nutils - *turek-hron-fsi3_fluid-nutils_solid-nutils + - *two-scale-heat-conduction_macro-nutils_micro-nutils # A selection of tests that cover a wide range of main features, meant for quicker CI executions precice: