Add per-test fieldcompare tolerance and skip_compare (#837)

changelog-entries/847.md

@@ -0,0 +1 @@
+- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration closes #837.

tools/tests/README.md

@@ -25,7 +25,7 @@ The available test suites are found in [`tests.yaml`](https://github.com/precice
 - `precice` is a subset of cases that cover a range of preCICE features
 - `release` is for all available but some very long or known to fail tests
 - `extra` is for some longer tests
-- `expected-to-fail`, `selected`, and `system-tests-dev` for some special cases
+- `selected` and `system-tests-dev` for some special cases
 
 The `Use workflow from` is a default option of GitHub Actions that concerns the GHA workflow file itself.
 
@@ -106,6 +106,8 @@ fieldcompare dir precice-exports/ reference-results-unpacked/<case>/ \
              -rtol 3e-7
 ```
 
+The default relative tolerance is `3e-7`. Per-test overrides are available in `tests.yaml` via `tolerance` (passed to fieldcompare as `-rtol`). Set `skip_compare: true` to skip the comparison step and only verify that build and run succeed.
+
 The differences are only shown per file, and there is no global metric or other summary (see [related discussion in fieldcompare](https://gitlab.com/dglaeser/fieldcompare/-/work_items/69)).
 
 Alternatively, [visualize the `precice-exports/diff_*.vtu` in ParaView](https://precice.org/configuration-export.html#visualization-with-paraview).
@@ -333,4 +335,6 @@ A `GLOBAL_TIMEOUT` is used for all operations. Its default value is 600s (5min),
 
 Tests can define a different `timeout` in their `tests.yaml` entry, which applies to the running and results comparison steps.
 
+Tests can define a different `tolerance` in their `tests.yaml` entry, which applies to the fieldcompare step (relative tolerance, `-rtol`). The default is `3e-7`. Use `skip_compare: true` to skip fieldcompare entirely.
+
 </details>

tools/tests/docker-compose.field_compare.template.yaml

@@ -6,4 +6,4 @@ services:
     command:
       - /runs/{{ tutorial_folder }}/{{ precice_output_folder }}
       - /runs/{{ tutorial_folder }}/{{ reference_output_folder }}
-      - "-rtol 3e-7 --ignore-missing-reference-files --diff"
+      - "-rtol {{ tolerance }} --ignore-missing-reference-files --diff"

tools/tests/generate_reference_results.py

@@ -2,7 +2,7 @@
 from metadata_parser.metdata import Tutorials, ReferenceResult
 from systemtests.TestSuite import TestSuites
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT
+from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL
 from pathlib import Path
 from typing import List
 from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR
@@ -145,13 +145,18 @@ def main():
             max_times = test_suite.max_times.get(tutorial, [])
             mtw_list = test_suite.max_time_windows.get(tutorial, [])
             timeouts = test_suite.timeouts.get(tutorial, [])
+            tolerances = test_suite.tolerances.get(tutorial, [])
+            skip_compares = test_suite.skip_compares.get(tutorial, [])
             for i, (case, reference_result) in enumerate(zip(
                     test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
                 max_time = max_times[i] if i < len(max_times) else None
                 max_time_windows = mtw_list[i] if i < len(mtw_list) else None
                 timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT
+                tolerance = tolerances[i] if i < len(
+                    tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL
+                skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False
                 systemtests_to_run.add(
-                    Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout))
+                    Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare))
 
     reference_result_per_tutorial = {}
     current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

tools/tests/systemtests.py

@@ -2,7 +2,7 @@
 import argparse
 from pathlib import Path
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table
+from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL, display_systemtestresults_as_table
 from systemtests.TestSuite import TestSuites
 from metadata_parser.metdata import Tutorials, Case
 import logging
@@ -89,13 +89,19 @@ def _group_end() -> None:
                 max_times = test_suite.max_times.get(tutorial, [])
                 mtw_list = test_suite.max_time_windows.get(tutorial, [])
                 timeouts = test_suite.timeouts.get(tutorial, [])
+                tolerances = test_suite.tolerances.get(tutorial, [])
+                skip_compares = test_suite.skip_compares.get(tutorial, [])
                 for i, (case, reference_result) in enumerate(zip(
                         test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
                     max_time = max_times[i] if i < len(max_times) else None
                     max_time_windows = mtw_list[i] if i < len(mtw_list) else None
                     timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT
+                    tolerance = tolerances[i] if i < len(
+                        tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL
+                    skip_compare = skip_compares[i] if i < len(
+                        skip_compares) and skip_compares[i] is not None else False
                     systemtests_to_run.append(
-                        Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout))
+                        Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare))
 
     if not systemtests_to_run:
         raise RuntimeError("Did not find any Systemtests to execute.")

tools/tests/systemtests/Systemtest.py

@@ -21,6 +21,7 @@
 
 
 GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 600))
+DEFAULT_FIELDCOMPARE_RTOL = 3e-7
 SHORT_TIMEOUT = 10
 
 DIFF_RESULTS_DIR = "diff-results"
@@ -219,6 +220,8 @@ class Systemtest:
     max_time: float | None = None
     max_time_windows: int | None = None
     timeout: int = GLOBAL_TIMEOUT
+    tolerance: float = DEFAULT_FIELDCOMPARE_RTOL
+    skip_compare: bool = False
     params_to_use: Dict[str, str] = field(init=False)
     env: Dict[str, str] = field(init=False)
 
@@ -330,6 +333,7 @@ def __get_field_compare_compose_file(self):
             'tutorial_folder': self.tutorial_folder,
             'precice_output_folder': PRECICE_REL_OUTPUT_DIR,
             'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""),
+            'tolerance': self.tolerance,
         }
         jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR))
         template = jinja_env.get_template(
@@ -669,6 +673,15 @@ def _run_field_compare(self):
         elapsed_time = time.perf_counter() - time_start
         return FieldCompareResult(exit_code, stdout_data, stderr_data, self, elapsed_time)
 
+    def _log_skipped_fieldcompare(self) -> None:
+        log_sink = getattr(self, "_log_sink", None)
+        if log_sink is not None:
+            log_sink.begin_stage("compare")
+            log_sink.append_stdout(
+                f"(skipped: skip_compare=true, default rtol would be {DEFAULT_FIELDCOMPARE_RTOL})",
+                "compare",
+            )
+
     def __archive_fieldcompare_diffs(self) -> None:
         """
         Copy fieldcompare diff VTK files from precice-exports/ into diff-results/,
@@ -833,20 +846,26 @@ def run(self, run_directory: Path):
                 solver_time=docker_run_result.runtime,
                 fieldcompare_time=0)
 
-        fieldcompare_result = self._run_field_compare()
-        std_out.extend(fieldcompare_result.stdout_data)
-        std_err.extend(fieldcompare_result.stderr_data)
-        if fieldcompare_result.exit_code != 0:
-            self.__archive_fieldcompare_diffs()
-            logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed")
-            return SystemtestResult(
-                False,
-                std_out,
-                std_err,
-                self,
-                build_time=docker_build_result.runtime,
-                solver_time=docker_run_result.runtime,
-                fieldcompare_time=fieldcompare_result.runtime)
+        if self.skip_compare:
+            logging.info(f"Skipping fieldcompare for {self} (skip_compare=true)")
+            self._log_skipped_fieldcompare()
+            fieldcompare_time = 0.0
+        else:
+            fieldcompare_result = self._run_field_compare()
+            std_out.extend(fieldcompare_result.stdout_data)
+            std_err.extend(fieldcompare_result.stderr_data)
+            if fieldcompare_result.exit_code != 0:
+                self.__archive_fieldcompare_diffs()
+                logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed")
+                return SystemtestResult(
+                    False,
+                    std_out,
+                    std_err,
+                    self,
+                    build_time=docker_build_result.runtime,
+                    solver_time=docker_run_result.runtime,
+                    fieldcompare_time=fieldcompare_result.runtime)
+            fieldcompare_time = fieldcompare_result.runtime
 
         # self.__cleanup()
         self._cleanup_docker_networks()
@@ -857,7 +876,7 @@ def run(self, run_directory: Path):
             self,
             build_time=docker_build_result.runtime,
             solver_time=docker_run_result.runtime,
-            fieldcompare_time=fieldcompare_result.runtime)
+            fieldcompare_time=fieldcompare_time)
 
     def run_for_reference_results(self, run_directory: Path):
         """

tools/tests/systemtests/TestSuite.py

@@ -13,6 +13,8 @@ class TestSuite:
     max_times: Dict[Tutorial, list] = field(default_factory=dict)
     max_time_windows: Dict[Tutorial, list] = field(default_factory=dict)
     timeouts: Dict[Tutorial, List] = field(default_factory=dict)
+    tolerances: Dict[Tutorial, list] = field(default_factory=dict)
+    skip_compares: Dict[Tutorial, list] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return_string = f"Test suite: {self.name} contains:"
@@ -54,6 +56,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                 max_times_of_tutorial = {}
                 max_time_windows_of_tutorial = {}
                 timeouts_of_tutorial = {}
+                tolerances_of_tutorial = {}
+                skip_compares_of_tutorial = {}
                 # iterate over tutorials:
                 for tutorial_case in test_suites_raw[test_suite_name]['tutorials']:
                     tutorial = parsed_tutorials.get_by_path(tutorial_case['path'])
@@ -66,6 +70,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                         max_times_of_tutorial[tutorial] = []
                         max_time_windows_of_tutorial[tutorial] = []
                         timeouts_of_tutorial[tutorial] = []
+                        tolerances_of_tutorial[tutorial] = []
+                        skip_compares_of_tutorial[tutorial] = []
 
                     all_case_combinations = tutorial.case_combinations
                     case_combination_requested = CaseCombination.from_string_list(
@@ -91,12 +97,34 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                                 f"(value: {timeout_value}) in tutorial '{tutorial}'."
                             )
                         timeouts_of_tutorial[tutorial].append(timeout_value)
+
+                        tolerance_value = tutorial_case.get('tolerance', None)
+                        if tolerance_value is not None:
+                            if isinstance(tolerance_value, str):
+                                try:
+                                    tolerance_value = float(tolerance_value)
+                                except ValueError as exc:
+                                    raise ValueError(
+                                        f"tolerance must be a positive number, got {tolerance_value!r}") from exc
+                            if not isinstance(tolerance_value, (int, float)) or tolerance_value <= 0:
+                                raise ValueError(
+                                    f"tolerance must be a positive number, got {tolerance_value!r}")
+                        tolerances_of_tutorial[tutorial].append(tolerance_value)
+
+                        skip_compare_value = tutorial_case.get('skip_compare', None)
+                        if skip_compare_value is not None and not isinstance(skip_compare_value, bool):
+                            raise TypeError(
+                                f"Expected 'skip_compare' to be a boolean or None, but got "
+                                f"{type(skip_compare_value).__name__} (value: {skip_compare_value}) "
+                                f"in tutorial '{tutorial}'."
+                            )
+                        skip_compares_of_tutorial[tutorial].append(skip_compare_value)
                     else:
                         raise Exception(
                             f"Could not find the case combination {tutorial_case['case_combination']} in the current metadata of tutorial {tutorial.name}, or it does not define all necessary participants.")
 
                 testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial,
-                                            reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial))
+                                            reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial, tolerances_of_tutorial, skip_compares_of_tutorial))
 
         return cls(testsuites)
 

tools/tests/tests.yaml

@@ -114,7 +114,8 @@ test_suites:
           - fluid-openfoam
           - solid-fenics
         max_time_windows: 1
-        reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz # Too small values, expected to fail the comparisons.
+        tolerance: 1e-2
+        reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz
 
   flow-around-controlled-moving-cylinder:
     tutorials:
@@ -487,14 +488,16 @@ test_suites:
         case_combination:
           - macro-dumux
           - micro-dumux
-        reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz # Too small values, expected to fail the comparisons.
+        tolerance: 1e-2
+        reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz
       - &two-scale-heat-conduction_macro-nutils_micro-nutils
         path: two-scale-heat-conduction
         case_combination:
           - macro-nutils
           - micro-nutils
         max_time: 0.05
-        reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz # Too small values, expected to fail the comparisons.
+        tolerance: 1e-2
+        reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz
 
   volume-coupled-diffusion:
     tutorials:
@@ -605,11 +608,6 @@ test_suites:
       - *perpendicular-flap_fluid-openfoam_solid-nutils
       - *turek-hron-fsi3_fluid-nutils_solid-nutils
 
-  expected-to-fail:
-    tutorials:
-      - *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare
-      - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare
-
   # A selection of tests that cover a wide range of main features, meant for quicker CI executions
   precice:
     tutorials:
@@ -653,9 +651,7 @@ test_suites:
       - *perpendicular-flap_fluid-openfoam_solid-fenics
       - *perpendicular-flap_fluid-su2_solid-fenics
       - *volume-coupled-diffusion_source-fenics_drain-fenics
-
-      # Excluded:
-      # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare
+      - *elastic-tube-3d_fluid-openfoam_solid-fenics
 
   fenicsx-adapter:
     tutorials:
@@ -678,8 +674,8 @@ test_suites:
 
   micro-manager:
     tutorials:
-      - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare
-      - *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare
+      - *two-scale-heat-conduction_macro-dumux_micro-dumux
+      - *two-scale-heat-conduction_macro-nutils_micro-nutils
 
   nutils-adapter: # Not a repository
     tutorials:
@@ -694,9 +690,7 @@ test_suites:
       - *turek-hron-fsi3_fluid-nutils_solid-nutils
       - *volume-coupled-flow_fluid-openfoam_source-nutils
       - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam
-
-      # Excluded:
-      # *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare
+      - *two-scale-heat-conduction_macro-nutils_micro-nutils
 
   openfoam-adapter:
     tutorials:
@@ -729,9 +723,7 @@ test_suites:
       - *quickstart_openfoam_cpp
       - *volume-coupled-flow_fluid-openfoam_source-nutils
       - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam
-
-      # Excluded:
-      # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare
+      - *elastic-tube-3d_fluid-openfoam_solid-fenics
 
   su2-adapter:
     tutorials: