From d43708298bd378e4744822d6b78c8498b9fa861f Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 17 Jun 2026 14:08:33 +0530 Subject: [PATCH 1/3] Add per-test fieldcompare tolerance and skip_compare (#837) Allow overriding fieldcompare rtol via tests.yaml and optionally skip the compare step. Re-enable previously quarantined tests with tolerance 1e-2 for near-zero exported fields. --- changelog-entries/837.md | 1 + tools/tests/README.md | 6 ++- ...docker-compose.field_compare.template.yaml | 2 +- tools/tests/generate_reference_results.py | 8 ++- tools/tests/systemtests.py | 8 ++- tools/tests/systemtests/Systemtest.py | 49 +++++++++++++------ tools/tests/systemtests/TestSuite.py | 30 +++++++++++- tools/tests/tests.yaml | 30 +++++------- 8 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 changelog-entries/837.md diff --git a/changelog-entries/837.md b/changelog-entries/837.md new file mode 100644 index 000000000..6b8ba6e1d --- /dev/null +++ b/changelog-entries/837.md @@ -0,0 +1 @@ +- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration ([#837](https://github.com/precice/tutorials/issues/837)). diff --git a/tools/tests/README.md b/tools/tests/README.md index 620b76164..4d0d62088 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -25,7 +25,7 @@ The available test suites are found in [`tests.yaml`](https://github.com/precice - `precice` is a subset of cases that cover a range of preCICE features - `release` is for all available but some very long or known to fail tests - `extra` is for some longer tests -- `expected-to-fail`, `selected`, and `system-tests-dev` for some special cases +- `selected` and `system-tests-dev` for some special cases The `Use workflow from` is a default option of GitHub Actions that concerns the GHA workflow file itself. @@ -106,6 +106,8 @@ fieldcompare dir precice-exports/ reference-results-unpacked// \ -rtol 3e-7 ``` +The default relative tolerance is `3e-7`. Per-test overrides are available in `tests.yaml` via `tolerance` (passed to fieldcompare as `-rtol`). Set `skip_compare: true` to skip the comparison step and only verify that build and run succeed. + The differences are only shown per file, and there is no global metric or other summary (see [related discussion in fieldcompare](https://gitlab.com/dglaeser/fieldcompare/-/work_items/69)). Alternatively, [visualize the `precice-exports/diff_*.vtu` in ParaView](https://precice.org/configuration-export.html#visualization-with-paraview). @@ -333,4 +335,6 @@ A `GLOBAL_TIMEOUT` is used for all operations. Its default value is 600s (5min), Tests can define a different `timeout` in their `tests.yaml` entry, which applies to the running and results comparison steps. +Tests can define a different `tolerance` in their `tests.yaml` entry, which applies to the fieldcompare step (relative tolerance, `-rtol`). The default is `3e-7`. Use `skip_compare: true` to skip fieldcompare entirely. + diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 8f6dd5e19..65590ab61 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -6,4 +6,4 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} - - "-rtol 3e-7 --ignore-missing-reference-files --diff" + - "-rtol {{ tolerance }} --ignore-missing-reference-files --diff" diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index d7470c8a6..698d11c85 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -2,7 +2,7 @@ from metadata_parser.metdata import Tutorials, ReferenceResult from systemtests.TestSuite import TestSuites from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL from pathlib import Path from typing import List from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR @@ -145,13 +145,17 @@ def main(): max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) timeouts = test_suite.timeouts.get(tutorial, []) + tolerances = test_suite.tolerances.get(tutorial, []) + skip_compares = test_suite.skip_compares.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT + tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 3183b0ca6..90c6bac62 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -2,7 +2,7 @@ import argparse from pathlib import Path from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, DEFAULT_FIELDCOMPARE_RTOL, display_systemtestresults_as_table from systemtests.TestSuite import TestSuites from metadata_parser.metdata import Tutorials, Case import logging @@ -89,13 +89,17 @@ def _group_end() -> None: max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) timeouts = test_suite.timeouts.get(tutorial, []) + tolerances = test_suite.tolerances.get(tutorial, []) + skip_compares = test_suite.skip_compares.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT + tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 932b6b944..f2a6bea00 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -21,6 +21,7 @@ GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 600)) +DEFAULT_FIELDCOMPARE_RTOL = 3e-7 SHORT_TIMEOUT = 10 DIFF_RESULTS_DIR = "diff-results" @@ -219,6 +220,8 @@ class Systemtest: max_time: float | None = None max_time_windows: int | None = None timeout: int = GLOBAL_TIMEOUT + tolerance: float = DEFAULT_FIELDCOMPARE_RTOL + skip_compare: bool = False params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -330,6 +333,7 @@ def __get_field_compare_compose_file(self): 'tutorial_folder': self.tutorial_folder, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, 'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""), + 'tolerance': self.tolerance, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template( @@ -669,6 +673,15 @@ def _run_field_compare(self): elapsed_time = time.perf_counter() - time_start return FieldCompareResult(exit_code, stdout_data, stderr_data, self, elapsed_time) + def _log_skipped_fieldcompare(self) -> None: + log_sink = getattr(self, "_log_sink", None) + if log_sink is not None: + log_sink.begin_stage("compare") + log_sink.append_stdout( + f"(skipped: skip_compare=true, default rtol would be {DEFAULT_FIELDCOMPARE_RTOL})", + "compare", + ) + def __archive_fieldcompare_diffs(self) -> None: """ Copy fieldcompare diff VTK files from precice-exports/ into diff-results/, @@ -833,20 +846,26 @@ def run(self, run_directory: Path): solver_time=docker_run_result.runtime, fieldcompare_time=0) - fieldcompare_result = self._run_field_compare() - std_out.extend(fieldcompare_result.stdout_data) - std_err.extend(fieldcompare_result.stderr_data) - if fieldcompare_result.exit_code != 0: - self.__archive_fieldcompare_diffs() - logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") - return SystemtestResult( - False, - std_out, - std_err, - self, - build_time=docker_build_result.runtime, - solver_time=docker_run_result.runtime, - fieldcompare_time=fieldcompare_result.runtime) + if self.skip_compare: + logging.info(f"Skipping fieldcompare for {self} (skip_compare=true)") + self._log_skipped_fieldcompare() + fieldcompare_time = 0.0 + else: + fieldcompare_result = self._run_field_compare() + std_out.extend(fieldcompare_result.stdout_data) + std_err.extend(fieldcompare_result.stderr_data) + if fieldcompare_result.exit_code != 0: + self.__archive_fieldcompare_diffs() + logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=fieldcompare_result.runtime) + fieldcompare_time = fieldcompare_result.runtime # self.__cleanup() self._cleanup_docker_networks() @@ -857,7 +876,7 @@ def run(self, run_directory: Path): self, build_time=docker_build_result.runtime, solver_time=docker_run_result.runtime, - fieldcompare_time=fieldcompare_result.runtime) + fieldcompare_time=fieldcompare_time) def run_for_reference_results(self, run_directory: Path): """ diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index a498ca4bd..a561e9ddf 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -13,6 +13,8 @@ class TestSuite: max_times: Dict[Tutorial, list] = field(default_factory=dict) max_time_windows: Dict[Tutorial, list] = field(default_factory=dict) timeouts: Dict[Tutorial, List] = field(default_factory=dict) + tolerances: Dict[Tutorial, list] = field(default_factory=dict) + skip_compares: Dict[Tutorial, list] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -54,6 +56,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_times_of_tutorial = {} max_time_windows_of_tutorial = {} timeouts_of_tutorial = {} + tolerances_of_tutorial = {} + skip_compares_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -66,6 +70,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_times_of_tutorial[tutorial] = [] max_time_windows_of_tutorial[tutorial] = [] timeouts_of_tutorial[tutorial] = [] + tolerances_of_tutorial[tutorial] = [] + skip_compares_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -91,12 +97,34 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): f"(value: {timeout_value}) in tutorial '{tutorial}'." ) timeouts_of_tutorial[tutorial].append(timeout_value) + + tolerance_value = tutorial_case.get('tolerance', None) + if tolerance_value is not None: + if isinstance(tolerance_value, str): + try: + tolerance_value = float(tolerance_value) + except ValueError as exc: + raise ValueError( + f"tolerance must be a positive number, got {tolerance_value!r}") from exc + if not isinstance(tolerance_value, (int, float)) or tolerance_value <= 0: + raise ValueError( + f"tolerance must be a positive number, got {tolerance_value!r}") + tolerances_of_tutorial[tutorial].append(tolerance_value) + + skip_compare_value = tutorial_case.get('skip_compare', None) + if skip_compare_value is not None and not isinstance(skip_compare_value, bool): + raise TypeError( + f"Expected 'skip_compare' to be a boolean or None, but got " + f"{type(skip_compare_value).__name__} (value: {skip_compare_value}) " + f"in tutorial '{tutorial}'." + ) + skip_compares_of_tutorial[tutorial].append(skip_compare_value) else: raise Exception( f"Could not find the case combination {tutorial_case['case_combination']} in the current metadata of tutorial {tutorial.name}, or it does not define all necessary participants.") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial)) + reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial, tolerances_of_tutorial, skip_compares_of_tutorial)) return cls(testsuites) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 70faade1d..926193ded 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -114,7 +114,8 @@ test_suites: - fluid-openfoam - solid-fenics max_time_windows: 1 - reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz # Too small values, expected to fail the comparisons. + tolerance: 1e-2 + reference_result: ./elastic-tube-3d/reference-results/fluid-openfoam_solid-fenics.tar.gz flow-around-controlled-moving-cylinder: tutorials: @@ -487,14 +488,16 @@ test_suites: case_combination: - macro-dumux - micro-dumux - reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz # Too small values, expected to fail the comparisons. + tolerance: 1e-2 + reference_result: ./two-scale-heat-conduction/reference-results/macro-dumux_micro-dumux.tar.gz - &two-scale-heat-conduction_macro-nutils_micro-nutils path: two-scale-heat-conduction case_combination: - macro-nutils - micro-nutils max_time: 0.05 - reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz # Too small values, expected to fail the comparisons. + tolerance: 1e-2 + reference_result: ./two-scale-heat-conduction/reference-results/macro-nutils_micro-nutils.tar.gz volume-coupled-diffusion: tutorials: @@ -605,11 +608,6 @@ test_suites: - *perpendicular-flap_fluid-openfoam_solid-nutils - *turek-hron-fsi3_fluid-nutils_solid-nutils - expected-to-fail: - tutorials: - - *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare - - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare - # A selection of tests that cover a wide range of main features, meant for quicker CI executions precice: tutorials: @@ -653,9 +651,7 @@ test_suites: - *perpendicular-flap_fluid-openfoam_solid-fenics - *perpendicular-flap_fluid-su2_solid-fenics - *volume-coupled-diffusion_source-fenics_drain-fenics - - # Excluded: - # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare + - *elastic-tube-3d_fluid-openfoam_solid-fenics fenicsx-adapter: tutorials: @@ -678,8 +674,8 @@ test_suites: micro-manager: tutorials: - - *two-scale-heat-conduction_macro-dumux_micro-dumux # too small values to compare - - *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare + - *two-scale-heat-conduction_macro-dumux_micro-dumux + - *two-scale-heat-conduction_macro-nutils_micro-nutils nutils-adapter: # Not a repository tutorials: @@ -694,9 +690,7 @@ test_suites: - *turek-hron-fsi3_fluid-nutils_solid-nutils - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - - # Excluded: - # *two-scale-heat-conduction_macro-nutils_micro-nutils # too small values to compare + - *two-scale-heat-conduction_macro-nutils_micro-nutils openfoam-adapter: tutorials: @@ -729,9 +723,7 @@ test_suites: - *quickstart_openfoam_cpp - *volume-coupled-flow_fluid-openfoam_source-nutils - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - - # Excluded: - # *elastic-tube-3d_fluid-openfoam_solid-fenics # too small values to compare + - *elastic-tube-3d_fluid-openfoam_solid-fenics su2-adapter: tutorials: From 98b91f661382995a5b7a5aadb9d217d332bcff88 Mon Sep 17 00:00:00 2001 From: Pranjal Date: Wed, 17 Jun 2026 14:42:28 +0530 Subject: [PATCH 2/3] Update changelog entry for issue #837 --- changelog-entries/837.md | 1 - changelog-entries/847.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog-entries/837.md create mode 100644 changelog-entries/847.md diff --git a/changelog-entries/837.md b/changelog-entries/837.md deleted file mode 100644 index 6b8ba6e1d..000000000 --- a/changelog-entries/837.md +++ /dev/null @@ -1 +0,0 @@ -- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration ([#837](https://github.com/precice/tutorials/issues/837)). diff --git a/changelog-entries/847.md b/changelog-entries/847.md new file mode 100644 index 000000000..a0ac31f1a --- /dev/null +++ b/changelog-entries/847.md @@ -0,0 +1 @@ +- Add optional `tolerance` and `skip_compare` fields to `tests.yaml` for per-test fieldcompare configuration closes #837. From da376f110b767864eec67b0d2373bf893613ff38 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Wed, 17 Jun 2026 14:45:32 +0530 Subject: [PATCH 3/3] Fix autopep8 line length in system test runners Wrap long tolerance/skip_compare lines so check_style pre-commit passes. --- tools/tests/generate_reference_results.py | 3 ++- tools/tests/systemtests.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 698d11c85..49decc7a7 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -152,7 +152,8 @@ def main(): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT - tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + tolerance = tolerances[i] if i < len( + tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False systemtests_to_run.add( Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare)) diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 90c6bac62..61c8273bc 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -96,8 +96,10 @@ def _group_end() -> None: max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT - tolerance = tolerances[i] if i < len(tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL - skip_compare = skip_compares[i] if i < len(skip_compares) and skip_compares[i] is not None else False + tolerance = tolerances[i] if i < len( + tolerances) and tolerances[i] is not None else DEFAULT_FIELDCOMPARE_RTOL + skip_compare = skip_compares[i] if i < len( + skip_compares) and skip_compares[i] is not None else False systemtests_to_run.append( Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, tolerance=tolerance, skip_compare=skip_compare))