diff --git a/changelog-entries/847.md b/changelog-entries/847.md new file mode 100644 index 000000000..1b9b516da --- /dev/null +++ b/changelog-entries/847.md @@ -0,0 +1 @@ +- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration [#847](https://github.com/precice/tutorials/pull/847) diff --git a/tools/tests/README.md b/tools/tests/README.md index 28cf367d5..e9807c7a1 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -25,7 +25,7 @@ The available test suites are found in [`tests.yaml`](https://github.com/precice - `precice` is a subset of cases that cover a range of preCICE features - `release` is for all available but some very long or known to fail tests - `extra` is for some longer tests -- `expected-to-fail`, `selected`, and `system-tests-dev` for some special cases +- `selected` and `system-tests-dev` for some special cases The `Use workflow from` is a default option of GitHub Actions that concerns the GHA workflow file itself. @@ -107,6 +107,8 @@ fieldcompare dir precice-exports/ reference-results-unpacked// \ -rtol 3e-7 ``` +The default relative tolerance (`-rtol`) is `3e-7`. Per-test overrides are possible in `tests.yaml` (e.g., `tolerance: 1e-2`). Set `skip_compare: true` to skip the comparison step and only verify that the build and run steps succeed. + The differences are only shown per file, and there is no global metric or other summary (see [related discussion in fieldcompare](https://gitlab.com/dglaeser/fieldcompare/-/work_items/69)). Alternatively, [visualize the `precice-exports/diff_*.vtu` in ParaView](https://precice.org/configuration-export.html#visualization-with-paraview). @@ -336,4 +338,6 @@ The build and the run/compare steps use separate timeouts. **Run and compare:** Each test in `tests.yaml` may set a `timeout` (default: 180s (3min)), which applies to the running and results comparison steps only. You can override the default via the `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable. +Tests can define a different `tolerance` in their `tests.yaml` entry, which applies to the fieldcompare step (relative tolerance, `-rtol`). The default is `3e-7`. Use `skip_compare: true` to skip fieldcompare entirely. + diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 8f6dd5e19..65590ab61 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -6,4 +6,4 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} - - "-rtol 3e-7 --ignore-missing-reference-files --diff" + - "-rtol {{ tolerance }} --ignore-missing-reference-files --diff" diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 3183b0ca6..61c8273bc 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -2,7 +2,7 @@ import argparse from pathlib import Path from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL, display_systemtestresults_as_table from systemtests.TestSuite import TestSuites from metadata_parser.metdata import Tutorials, Case import logging @@ -89,13 +89,19 @@ def _group_end() -> None: max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) timeouts = test_suite.timeouts.get(tutorial, []) + tolerances = test_suite.tolerances.get(tutorial, []) + skip_compares = test_suite.skip_compares.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT + tolerance = tolerances[i] if i < len( + tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + skip_compare = skip_compares[i] if i < len( + skip_compares) and skip_compares[i] is not None else False systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 3ffb74df0..d65260553 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -24,6 +24,7 @@ GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 180)) DEFAULT_BUILD_TIMEOUT = int( os.environ.get("PRECICE_SYSTEMTESTS_BUILD_TIMEOUT", 480)) +DEFAULT_FIELDCOMPARE_RTOL = 3e-7 SHORT_TIMEOUT = 10 DIFF_RESULTS_DIR = "diff-results" @@ -223,6 +224,8 @@ class Systemtest: max_time: float | None = None max_time_windows: int | None = None timeout: int = GLOBAL_TIMEOUT + tolerance: float = DEFAULT_FIELDCOMPARE_RTOL + skip_compare: bool = False params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -355,6 +358,7 @@ def __get_field_compare_compose_file(self): 'tutorial_folder': self.tutorial_folder, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, 'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""), + 'tolerance': self.tolerance, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template( @@ -991,20 +995,25 @@ def run(self, run_directory: Path): solver_time=docker_run_result.runtime, fieldcompare_time=0) - fieldcompare_result = self._run_field_compare() - std_out.extend(fieldcompare_result.stdout_data) - std_err.extend(fieldcompare_result.stderr_data) - if fieldcompare_result.exit_code != 0: - self.__archive_fieldcompare_diffs() - logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") - return SystemtestResult( - False, - std_out, - std_err, - self, - build_time=docker_build_result.runtime, - solver_time=docker_run_result.runtime, - fieldcompare_time=fieldcompare_result.runtime) + if self.skip_compare: + logging.info(f"Skipping fieldcompare for {self} (skip_compare=true)") + fieldcompare_time = 0.0 + else: + fieldcompare_result = self._run_field_compare() + std_out.extend(fieldcompare_result.stdout_data) + std_err.extend(fieldcompare_result.stderr_data) + if fieldcompare_result.exit_code != 0: + self.__archive_fieldcompare_diffs() + logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=fieldcompare_result.runtime) + fieldcompare_time = fieldcompare_result.runtime self.__archive_iterations_logs() if not self.__compare_iterations_hashes(): @@ -1029,7 +1038,7 @@ def run(self, run_directory: Path): self, build_time=docker_build_result.runtime, solver_time=docker_run_result.runtime, - fieldcompare_time=fieldcompare_result.runtime) + fieldcompare_time=fieldcompare_time) def run_for_reference_results(self, run_directory: Path): """ diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index a498ca4bd..a561e9ddf 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -13,6 +13,8 @@ class TestSuite: max_times: Dict[Tutorial, list] = field(default_factory=dict) max_time_windows: Dict[Tutorial, list] = field(default_factory=dict) timeouts: Dict[Tutorial, List] = field(default_factory=dict) + tolerances: Dict[Tutorial, list] = field(default_factory=dict) + skip_compares: Dict[Tutorial, list] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -54,6 +56,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_times_of_tutorial = {} max_time_windows_of_tutorial = {} timeouts_of_tutorial = {} + tolerances_of_tutorial = {} + skip_compares_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -66,6 +70,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_times_of_tutorial[tutorial] = [] max_time_windows_of_tutorial[tutorial] = [] timeouts_of_tutorial[tutorial] = [] + tolerances_of_tutorial[tutorial] = [] + skip_compares_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -91,12 +97,34 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): f"(value: {timeout_value}) in tutorial '{tutorial}'." ) timeouts_of_tutorial[tutorial].append(timeout_value) + + tolerance_value = tutorial_case.get('tolerance', None) + if tolerance_value is not None: + if isinstance(tolerance_value, str): + try: + tolerance_value = float(tolerance_value) + except ValueError as exc: + raise ValueError( + f"tolerance must be a positive number, got {tolerance_value!r}") from exc + if not isinstance(tolerance_value, (int, float)) or tolerance_value <= 0: + raise ValueError( + f"tolerance must be a positive number, got {tolerance_value!r}") + tolerances_of_tutorial[tutorial].append(tolerance_value) + + skip_compare_value = tutorial_case.get('skip_compare', None) + if skip_compare_value is not None and not isinstance(skip_compare_value, bool): + raise TypeError( + f"Expected 'skip_compare' to be a boolean or None, but got " + f"{type(skip_compare_value).__name__} (value: {skip_compare_value}) " + f"in tutorial '{tutorial}'." + ) + skip_compares_of_tutorial[tutorial].append(skip_compare_value) else: raise Exception( f"Could not find the case combination {tutorial_case['case_combination']} in the current metadata of tutorial {tutorial.name}, or it does not define all necessary participants.") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial)) + reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial, tolerances_of_tutorial, skip_compares_of_tutorial)) return cls(testsuites) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 97f537a92..4901ecfa0 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -121,7 +121,9 @@ test_suites: - fluid-openfoam - solid-fenics max_time_windows: 1 - reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz # Too small values, expected to fail the comparisons. + tolerance: 1e-2 # Some values are too close to zero + skip_compare: true # Comparison takes too + reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz flow-around-controlled-moving-cylinder: tutorials: @@ -509,14 +511,16 @@ test_suites: case_combination: - macro-dumux - micro-dumux - reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz # Too small values, expected to fail the comparisons. + skip_compare: true # Values too close to zero + reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz - &two-scale-heat-conduction_macro-nutils_micro-nutils path: two-scale-heat-conduction case_combination: - macro-nutils - micro-nutils max_time: 0.05 - reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz # Too small values, expected to fail the comparisons. + skip_compare: true # Values too close to zero + reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz volume-coupled-diffusion: tutorials: @@ -576,6 +580,7 @@ test_suites: - *elastic-tube-1d_fluid-fortran-module_solid-fortran-module - *elastic-tube-1d_fluid-python_solid-python - *elastic-tube-3d_fluid-openfoam_solid-calculix + - *elastic-tube-3d_fluid-openfoam_solid-fenics - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python - *flow-over-heated-plate_fluid-openfoam_solid-fenics - *flow-over-heated-plate_fluid-openfoam_solid-fenicsx @@ -616,6 +621,7 @@ test_suites: - *quickstart_openfoam_cpp - *resonant-circuit_capacitor-python_coil-python - *turek-hron-fsi3_fluid-openfoam_solid-dealii + - *two-scale-heat-conduction_macro-dumux_micro-dumux - *volume-coupled-diffusion_source-fenics_drain-fenics - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam @@ -629,11 +635,7 @@ test_suites: - *perpendicular-flap_fluid-nutils_solid-calculix - *perpendicular-flap_fluid-openfoam_solid-nutils - *turek-hron-fsi3_fluid-nutils_solid-nutils - - expected-to-fail: - tutorials: - - *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare - - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare + - *two-scale-heat-conduction_macro-nutils_micro-nutils # A selection of tests that cover a wide range of main features, meant for quicker CI executions precice: @@ -671,6 +673,7 @@ test_suites: fenics-adapter: tutorials: - *channel-transport-reaction_fluid-fenics_chemical-fenics + - *elastic-tube-3d_fluid-openfoam_solid-fenics - *flow-over-heated-plate_fluid-openfoam_solid-fenics - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics - *partitioned-heat-conduction-complex_dirichlet-fenics_neumann-fenics @@ -679,9 +682,6 @@ test_suites: - *perpendicular-flap_fluid-su2_solid-fenics - *volume-coupled-diffusion_source-fenics_drain-fenics - # Excluded: - # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare - fenicsx-adapter: tutorials: - *flow-over-heated-plate_fluid-openfoam_solid-fenicsx @@ -703,8 +703,8 @@ test_suites: micro-manager: tutorials: - - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare - - *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare + - *two-scale-heat-conduction_macro-dumux_micro-dumux + - *two-scale-heat-conduction_macro-nutils_micro-nutils nutils-adapter: # Not a repository tutorials: @@ -717,12 +717,10 @@ test_suites: - *partitioned-pipe-multiscale_fluid1d-left-nutils_fluid3d-right-openfoam - *perpendicular-flap_fluid-nutils_solid-calculix - *turek-hron-fsi3_fluid-nutils_solid-nutils + - *two-scale-heat-conduction_macro-nutils_micro-nutils - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - # Excluded: - # *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare - openfoam-adapter: tutorials: - *breaking-dam-2d_fluid-openfoam_solid-calculix @@ -730,6 +728,7 @@ test_suites: - *channel-transport_fluid-openfoam_transport-openfoam - *channel-transport-particles_fluid-openfoam_particles-mercurydpm - *elastic-tube-3d_fluid-openfoam_solid-calculix + - *elastic-tube-3d_fluid-openfoam_solid-fenics - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python - *flow-over-heated-plate_fluid-openfoam_solid-fenics - *flow-over-heated-plate_fluid-openfoam_solid-nutils @@ -756,9 +755,6 @@ test_suites: - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - # Excluded: - # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare - su2-adapter: tutorials: - *flow-over-heated-plate_fluid-su2_solid-openfoam diff --git a/two-scale-heat-conduction/metadata.yaml b/two-scale-heat-conduction/metadata.yaml index 6c7c1dafb..bb0f361cd 100644 --- a/two-scale-heat-conduction/metadata.yaml +++ b/two-scale-heat-conduction/metadata.yaml @@ -16,17 +16,17 @@ cases: macro-nutils: participant: Macro directory: ./macro-nutils - run: ./run.sh -l /home/precice/nutils + run: ./run.sh component: nutils-adapter micro-dumux: participant: Micro-Manager directory: ./micro-dumux - run: source /home/precice/venv/bin/activate && ./run.sh -l /home/precice/dumux + run: source /home/precice/venv/bin/activate && ./run.sh -s -l /home/precice/dumux component: dumux-adapter micro-nutils: participant: Micro-Manager directory: ./micro-nutils - run: source /home/precice/venv/bin/activate && ./run.sh -l /home/precice/nutils + run: source /home/precice/venv/bin/activate && ./run.sh -s component: nutils-adapter \ No newline at end of file