Add per-test fieldcompare tolerance and skip_compare

changelog-entries/847.md

@@ -0,0 +1 @@
+- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration [#847](https://github.com/precice/tutorials/pull/847)

tools/tests/README.md

@@ -25,7 +25,7 @@ The available test suites are found in [`tests.yaml`](https://github.com/precice
 - `precice` is a subset of cases that cover a range of preCICE features
 - `release` is for all available but some very long or known to fail tests
 - `extra` is for some longer tests
-- `expected-to-fail`, `selected`, and `system-tests-dev` for some special cases
+- `selected` and `system-tests-dev` for some special cases
 
 The `Use workflow from` is a default option of GitHub Actions that concerns the GHA workflow file itself.
 
@@ -107,6 +107,8 @@ fieldcompare dir precice-exports/ reference-results-unpacked/<case>/ \
              -rtol 3e-7
 ```
 
+The default relative tolerance (`-rtol`) is `3e-7`. Per-test overrides are possible in `tests.yaml` (e.g., `tolerance: 1e-2`). Set `skip_compare: true` to skip the comparison step and only verify that the build and run steps succeed.
+
 The differences are only shown per file, and there is no global metric or other summary (see [related discussion in fieldcompare](https://gitlab.com/dglaeser/fieldcompare/-/work_items/69)).
 
 Alternatively, [visualize the `precice-exports/diff_*.vtu` in ParaView](https://precice.org/configuration-export.html#visualization-with-paraview).
@@ -336,4 +338,6 @@ The build and the run/compare steps use separate timeouts.
 
 **Run and compare:** Each test in `tests.yaml` may set a `timeout` (default: 180s (3min)), which applies to the running and results comparison steps only. You can override the default via the `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable.
 
+Tests can define a different `tolerance` in their `tests.yaml` entry, which applies to the fieldcompare step (relative tolerance, `-rtol`). The default is `3e-7`. Use `skip_compare: true` to skip fieldcompare entirely.
+
 </details>

tools/tests/docker-compose.field_compare.template.yaml

@@ -6,4 +6,4 @@ services:
     command:
       - /runs/{{ tutorial_folder }}/{{ precice_output_folder }}
       - /runs/{{ tutorial_folder }}/{{ reference_output_folder }}
-      - "-rtol 3e-7 --ignore-missing-reference-files --diff"
+      - "-rtol {{ tolerance }} --ignore-missing-reference-files --diff"

tools/tests/systemtests.py

@@ -2,7 +2,7 @@
 import argparse
 from pathlib import Path
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table
+from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL, display_systemtestresults_as_table
 from systemtests.TestSuite import TestSuites
 from metadata_parser.metdata import Tutorials, Case
 import logging
@@ -89,13 +89,19 @@ def _group_end() -> None:
                 max_times = test_suite.max_times.get(tutorial, [])
                 mtw_list = test_suite.max_time_windows.get(tutorial, [])
                 timeouts = test_suite.timeouts.get(tutorial, [])
+                tolerances = test_suite.tolerances.get(tutorial, [])
+                skip_compares = test_suite.skip_compares.get(tutorial, [])
                 for i, (case, reference_result) in enumerate(zip(
                         test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
                     max_time = max_times[i] if i < len(max_times) else None
                     max_time_windows = mtw_list[i] if i < len(mtw_list) else None
                     timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT
+                    tolerance = tolerances[i] if i < len(
+                        tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL
+                    skip_compare = skip_compares[i] if i < len(
+                        skip_compares) and skip_compares[i] is not None else False
                     systemtests_to_run.append(
-                        Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout))
+                        Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare))
 
     if not systemtests_to_run:
         raise RuntimeError("Did not find any Systemtests to execute.")

tools/tests/systemtests/Systemtest.py

@@ -24,6 +24,7 @@
 GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 180))
 DEFAULT_BUILD_TIMEOUT = int(
     os.environ.get("PRECICE_SYSTEMTESTS_BUILD_TIMEOUT", 480))
+DEFAULT_FIELDCOMPARE_RTOL = 3e-7
 SHORT_TIMEOUT = 10
 
 DIFF_RESULTS_DIR = "diff-results"
@@ -223,6 +224,8 @@ class Systemtest:
     max_time: float | None = None
     max_time_windows: int | None = None
     timeout: int = GLOBAL_TIMEOUT
+    tolerance: float = DEFAULT_FIELDCOMPARE_RTOL
+    skip_compare: bool = False
     params_to_use: Dict[str, str] = field(init=False)
     env: Dict[str, str] = field(init=False)
 
@@ -355,6 +358,7 @@ def __get_field_compare_compose_file(self):
             'tutorial_folder': self.tutorial_folder,
             'precice_output_folder': PRECICE_REL_OUTPUT_DIR,
             'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""),
+            'tolerance': self.tolerance,
         }
         jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR))
         template = jinja_env.get_template(
@@ -991,20 +995,25 @@ def run(self, run_directory: Path):
                 solver_time=docker_run_result.runtime,
                 fieldcompare_time=0)
 
-        fieldcompare_result = self._run_field_compare()
-        std_out.extend(fieldcompare_result.stdout_data)
-        std_err.extend(fieldcompare_result.stderr_data)
-        if fieldcompare_result.exit_code != 0:
-            self.__archive_fieldcompare_diffs()
-            logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed")
-            return SystemtestResult(
-                False,
-                std_out,
-                std_err,
-                self,
-                build_time=docker_build_result.runtime,
-                solver_time=docker_run_result.runtime,
-                fieldcompare_time=fieldcompare_result.runtime)
+        if self.skip_compare:
+            logging.info(f"Skipping fieldcompare for {self} (skip_compare=true)")
+            fieldcompare_time = 0.0
+        else:
+            fieldcompare_result = self._run_field_compare()
+            std_out.extend(fieldcompare_result.stdout_data)
+            std_err.extend(fieldcompare_result.stderr_data)
+            if fieldcompare_result.exit_code != 0:
+                self.__archive_fieldcompare_diffs()
+                logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed")
+                return SystemtestResult(
+                    False,
+                    std_out,
+                    std_err,
+                    self,
+                    build_time=docker_build_result.runtime,
+                    solver_time=docker_run_result.runtime,
+                    fieldcompare_time=fieldcompare_result.runtime)
+            fieldcompare_time = fieldcompare_result.runtime
 
         self.__archive_iterations_logs()
         if not self.__compare_iterations_hashes():
@@ -1029,7 +1038,7 @@ def run(self, run_directory: Path):
             self,
             build_time=docker_build_result.runtime,
             solver_time=docker_run_result.runtime,
-            fieldcompare_time=fieldcompare_result.runtime)
+            fieldcompare_time=fieldcompare_time)
 
     def run_for_reference_results(self, run_directory: Path):
         """

tools/tests/systemtests/TestSuite.py

@@ -13,6 +13,8 @@ class TestSuite:
     max_times: Dict[Tutorial, list] = field(default_factory=dict)
     max_time_windows: Dict[Tutorial, list] = field(default_factory=dict)
     timeouts: Dict[Tutorial, List] = field(default_factory=dict)
+    tolerances: Dict[Tutorial, list] = field(default_factory=dict)
+    skip_compares: Dict[Tutorial, list] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return_string = f"Test suite: {self.name} contains:"
@@ -54,6 +56,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                 max_times_of_tutorial = {}
                 max_time_windows_of_tutorial = {}
                 timeouts_of_tutorial = {}
+                tolerances_of_tutorial = {}
+                skip_compares_of_tutorial = {}
                 # iterate over tutorials:
                 for tutorial_case in test_suites_raw[test_suite_name]['tutorials']:
                     tutorial = parsed_tutorials.get_by_path(tutorial_case['path'])
@@ -66,6 +70,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                         max_times_of_tutorial[tutorial] = []
                         max_time_windows_of_tutorial[tutorial] = []
                         timeouts_of_tutorial[tutorial] = []
+                        tolerances_of_tutorial[tutorial] = []
+                        skip_compares_of_tutorial[tutorial] = []
 
                     all_case_combinations = tutorial.case_combinations
                     case_combination_requested = CaseCombination.from_string_list(
@@ -91,12 +97,34 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                                 f"(value: {timeout_value}) in tutorial '{tutorial}'."
                             )
                         timeouts_of_tutorial[tutorial].append(timeout_value)
+
+                        tolerance_value = tutorial_case.get('tolerance', None)
+                        if tolerance_value is not None:
+                            if isinstance(tolerance_value, str):
+                                try:
+                                    tolerance_value = float(tolerance_value)
+                                except ValueError as exc:
+                                    raise ValueError(
+                                        f"tolerance must be a positive number, got {tolerance_value!r}") from exc
+                            if not isinstance(tolerance_value, (int, float)) or tolerance_value <= 0:
+                                raise ValueError(
+                                    f"tolerance must be a positive number, got {tolerance_value!r}")
+                        tolerances_of_tutorial[tutorial].append(tolerance_value)
+
+                        skip_compare_value = tutorial_case.get('skip_compare', None)
+                        if skip_compare_value is not None and not isinstance(skip_compare_value, bool):
+                            raise TypeError(
+                                f"Expected 'skip_compare' to be a boolean or None, but got "
+                                f"{type(skip_compare_value).__name__} (value: {skip_compare_value}) "
+                                f"in tutorial '{tutorial}'."
+                            )
+                        skip_compares_of_tutorial[tutorial].append(skip_compare_value)
                     else:
                         raise Exception(
                             f"Could not find the case combination {tutorial_case['case_combination']} in the current metadata of tutorial {tutorial.name}, or it does not define all necessary participants.")
 
                 testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial,
-                                            reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial))
+                                            reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial, tolerances_of_tutorial, skip_compares_of_tutorial))
 
         return cls(testsuites)
 

tools/tests/tests.yaml

@@ -121,7 +121,9 @@ test_suites:
           - fluid-openfoam
           - solid-fenics
         max_time_windows: 1
-        reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz # Too small values, expected to fail the comparisons.
+        tolerance: 1e-2    # Some values are too close to zero
+        skip_compare: true # Comparison takes too
+        reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz
 
   flow-around-controlled-moving-cylinder:
     tutorials:
@@ -509,14 +511,16 @@ test_suites:
         case_combination:
           - macro-dumux
           - micro-dumux
-        reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz # Too small values, expected to fail the comparisons.
+        skip_compare: true # Values too close to zero
+        reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz
       - &two-scale-heat-conduction_macro-nutils_micro-nutils
         path: two-scale-heat-conduction
         case_combination:
           - macro-nutils
           - micro-nutils
         max_time: 0.05
-        reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz # Too small values, expected to fail the comparisons.
+        skip_compare: true # Values too close to zero
+        reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz
 
   volume-coupled-diffusion:
     tutorials:
@@ -576,6 +580,7 @@ test_suites:
       - *elastic-tube-1d_fluid-fortran-module_solid-fortran-module
       - *elastic-tube-1d_fluid-python_solid-python
       - *elastic-tube-3d_fluid-openfoam_solid-calculix
+      - *elastic-tube-3d_fluid-openfoam_solid-fenics
       - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python
       - *flow-over-heated-plate_fluid-openfoam_solid-fenics
       - *flow-over-heated-plate_fluid-openfoam_solid-fenicsx
@@ -616,6 +621,7 @@ test_suites:
       - *quickstart_openfoam_cpp
       - *resonant-circuit_capacitor-python_coil-python
       - *turek-hron-fsi3_fluid-openfoam_solid-dealii
+      - *two-scale-heat-conduction_macro-dumux_micro-dumux
       - *volume-coupled-diffusion_source-fenics_drain-fenics
       - *volume-coupled-flow_fluid-openfoam_source-nutils
       - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam
@@ -629,11 +635,7 @@ test_suites:
       - *perpendicular-flap_fluid-nutils_solid-calculix
       - *perpendicular-flap_fluid-openfoam_solid-nutils
       - *turek-hron-fsi3_fluid-nutils_solid-nutils
-
-  expected-to-fail:
-    tutorials:
-      - *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare
-      - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare
+      - *two-scale-heat-conduction_macro-nutils_micro-nutils
 
   # A selection of tests that cover a wide range of main features, meant for quicker CI executions
   precice:
@@ -671,6 +673,7 @@ test_suites:
   fenics-adapter:
     tutorials:
       - *channel-transport-reaction_fluid-fenics_chemical-fenics
+      - *elastic-tube-3d_fluid-openfoam_solid-fenics
       - *flow-over-heated-plate_fluid-openfoam_solid-fenics
       - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics
       - *partitioned-heat-conduction-complex_dirichlet-fenics_neumann-fenics
@@ -679,9 +682,6 @@ test_suites:
       - *perpendicular-flap_fluid-su2_solid-fenics
       - *volume-coupled-diffusion_source-fenics_drain-fenics
 
-      # Excluded:
-      # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare
-
   fenicsx-adapter:
     tutorials:
       - *flow-over-heated-plate_fluid-openfoam_solid-fenicsx
@@ -703,8 +703,8 @@ test_suites:
 
   micro-manager:
     tutorials:
-      - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare
-      - *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare
+      - *two-scale-heat-conduction_macro-dumux_micro-dumux
+      - *two-scale-heat-conduction_macro-nutils_micro-nutils
 
   nutils-adapter: # Not a repository
     tutorials:
@@ -717,19 +717,18 @@ test_suites:
       - *partitioned-pipe-multiscale_fluid1d-left-nutils_fluid3d-right-openfoam
       - *perpendicular-flap_fluid-nutils_solid-calculix
       - *turek-hron-fsi3_fluid-nutils_solid-nutils
+      - *two-scale-heat-conduction_macro-nutils_micro-nutils
       - *volume-coupled-flow_fluid-openfoam_source-nutils
       - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam
 
-      # Excluded:
-      # *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare
-
   openfoam-adapter:
     tutorials:
       - *breaking-dam-2d_fluid-openfoam_solid-calculix
       - *channel-transport_fluid-openfoam_transport-nutils
       - *channel-transport_fluid-openfoam_transport-openfoam
       - *channel-transport-particles_fluid-openfoam_particles-mercurydpm
       - *elastic-tube-3d_fluid-openfoam_solid-calculix
+      - *elastic-tube-3d_fluid-openfoam_solid-fenics
       - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python
       - *flow-over-heated-plate_fluid-openfoam_solid-fenics
       - *flow-over-heated-plate_fluid-openfoam_solid-nutils
@@ -756,9 +755,6 @@ test_suites:
       - *volume-coupled-flow_fluid-openfoam_source-nutils
       - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam
 
-      # Excluded:
-      # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare
-
   su2-adapter:
     tutorials:
       - *flow-over-heated-plate_fluid-su2_solid-openfoam

two-scale-heat-conduction/metadata.yaml

@@ -16,17 +16,17 @@ cases:
   macro-nutils:
     participant: Macro
     directory: ./macro-nutils
-    run: ./run.sh -l /home/precice/nutils
+    run: ./run.sh
     component: nutils-adapter
 
   micro-dumux:
     participant: Micro-Manager
     directory: ./micro-dumux
-    run: source /home/precice/venv/bin/activate && ./run.sh -l /home/precice/dumux
+    run: source /home/precice/venv/bin/activate && ./run.sh -s -l /home/precice/dumux
     component: dumux-adapter
 
   micro-nutils:
     participant: Micro-Manager
     directory: ./micro-nutils
-    run: source /home/precice/venv/bin/activate && ./run.sh -l /home/precice/nutils
+    run: source /home/precice/venv/bin/activate && ./run.sh -s
     component: nutils-adapter