From ddd0eac05aa66cee2841983464123dbca5ac417a Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Thu, 21 May 2026 22:53:34 +0530 Subject: [PATCH 1/2] Make system test run directories portable for CI artifact replay Use relative Docker Compose paths, generate rerun_systemtest.sh in each run folder, and document replay from system_tests_run_*_full artifacts. Closes #387. --- changelog-entries/724.md | 1 + tools/tests/README.md | 51 ++++++++++++++++++- tools/tests/systemtests/Systemtest.py | 72 ++++++++++++++++++++++++--- 3 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 changelog-entries/724.md diff --git a/changelog-entries/724.md b/changelog-entries/724.md new file mode 100644 index 000000000..df4a54cbb --- /dev/null +++ b/changelog-entries/724.md @@ -0,0 +1 @@ +- System test run directories use relative Docker Compose paths and include a `rerun_systemtest.sh` script so CI artifacts can be downloaded and replayed locally (Closes #387). diff --git a/tools/tests/README.md b/tools/tests/README.md index bd935965b..eb519a543 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -104,9 +104,58 @@ In this case, building and running seems to work out, but the tests fail because ## Understanding what went wrong The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. -If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. +If this does not provide enough hints, the next step is to download the generated `system_tests_run___full` artifact (a smaller `_logs` archive contains only log files). Note that by default this will only be generated if the systemtests fail. Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: `system-tests-stderr.log` and `system-tests-stdout.log`. This can be a starting point for a further investigation. When fieldcompare runs with `--diff`, it writes VTK diff files under `precice-exports/`; if the comparison fails, those files are copied into a `diff-results/` subfolder in the same run directory (mirroring any subpaths under `precice-exports/`) so you can open them (e.g. in ParaView) to see where results differ from the reference. On successful comparisons, `diff-results/` is therefore absent. +### Re-running system tests from CI artifacts + +Download the **full** artifact from a failed (or manually uploaded) workflow run: + +`system_tests_run___full` + +The archive contains a `runs/` directory shared by all system tests from that run: + +```text +runs/ +├── tools/ # Dockerfiles and helpers (shared) +└── __/ # one folder per system test + ├── docker-compose.tutorial.yaml + ├── docker-compose.field_compare.yaml # if fieldcompare ran + ├── rerun_systemtest.sh + └── … +``` + +To re-run one test locally: + +1. Download and extract `system_tests_run___full.zip`. +2. Keep the `runs/` layout intact (the tutorial folder needs the sibling `tools/` directory): + + ```bash + unzip system_tests_run___full.zip + cd system_tests_run___full/runs + ls + cd __ + ``` + +3. In the tutorial folder you will find the copied tutorial, generated Docker Compose + files, and `rerun_systemtest.sh`. The shared `tools/` tree lives one level up in + `runs/tools/`. + +4. Re-run with Docker: + + ```bash + ./rerun_systemtest.sh # or: sh rerun_systemtest.sh + ``` + +The script rebuilds images, runs the tutorial containers, and (if +`docker-compose.field_compare.yaml` exists) runs fieldcompare with the same +`--exit-code-from` behavior as the CI runner. Compose paths are relative to the +tutorial folder (`..` points at the parent `runs/` directory), so you can relocate the +entire extracted `runs/` tree on any Linux host with Docker. + +Fieldcompare requires reference results in the artifact (unpacked by CI during the +original run) or you must unpack them manually before that step. + ## Adding new tests ### Adding tutorials diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 19d48ae7f..0692454e4 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -192,20 +192,37 @@ def __get_docker_services(self) -> Dict[str, str]: except Exception as exc: raise KeyError("Please specify a PLATFORM argument") from exc + # Use an absolute path here only for validation that the requested + # dockerfile context exists on the machine running the system tests. self.dockerfile_context = PRECICE_TESTS_DIR / "dockerfiles" / Path(plaform_requested) if not self.dockerfile_context.exists(): raise ValueError( f"The path {self.dockerfile_context.resolve()} resulting from argument PLATFORM={plaform_requested} could not be found in the system") def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) -> str: + # Inside the individual system test directory (`self.system_test_dir`) + # we copy a full `tools/` tree into the parent run directory + # (see __copy_tools). From the point of view of the system test + # directory we therefore need to go one level up to reach the + # shared `tools/` folder: + # /tools/tests/dockerfiles/ + # ^-------------^ parent of self.system_test_dir + dockerfile_context_relative = ( + Path("..") / "tools" / "tests" / "dockerfiles" / Path(plaform_requested) + ) + render_dict = { - 'run_directory': self.run_directory.resolve(), + # Use a relative path to the *parent* run directory so that + # containers still see /runs/ like before, + # while keeping the compose file independent of the CI + # runner's absolute paths. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'build_arguments': params_to_use, 'params': params_to_use, 'case_folder': case.path, 'run': case.run_cmd, - 'dockerfile_context': self.dockerfile_context, + 'dockerfile_context': dockerfile_context_relative, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template(case.component.template) @@ -220,12 +237,20 @@ def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) def __get_docker_compose_file(self): rendered_services = self.__get_docker_services() render_dict = { - 'run_directory': self.run_directory.resolve(), + # See __get_docker_services: keep the docker-compose file + # portable by referring to the parent run directory only. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'tutorial': self.tutorial.path.name, 'services': rendered_services, 'build_arguments': self.params_to_use, - 'dockerfile_context': self.dockerfile_context, + # The dockerfile_context value inside the templates is only + # used as a build context path and does not need to be + # absolute – it will be resolved relative to the system test + # directory. + 'dockerfile_context': ( + Path("..") / "tools" / "tests" / "dockerfiles" / Path(self.params_to_use.get("PLATFORM")) + ), 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) @@ -234,7 +259,10 @@ def __get_docker_compose_file(self): def __get_field_compare_compose_file(self): render_dict = { - 'run_directory': self.run_directory.resolve(), + # Fieldcompare should also use only relative paths from inside + # the system test directory so that the run directory can be + # moved and re-executed elsewhere. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, 'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""), @@ -498,9 +526,41 @@ def _build_docker(self): logging.debug(f"Building docker image for {self}") time_start = time.perf_counter() docker_compose_content = self.__get_docker_compose_file() - with open(self.system_test_dir / "docker-compose.tutorial.yaml", 'w') as file: + docker_compose_path = self.system_test_dir / "docker-compose.tutorial.yaml" + with open(docker_compose_path, 'w') as file: file.write(docker_compose_content) + # Provide a small helper script inside the system test directory so + # that a user downloading the corresponding `runs/` artifact can + # re-run the exact docker-compose setup locally without having to + # reconstruct the commands by hand. + rerun_script_path = self.system_test_dir / "rerun_systemtest.sh" + rerun_script_path.write_text( + "#!/usr/bin/env sh\n" + "set -e -u\n" + "\n" + "cd \"$(dirname \"$0\")\"\n" + "\n" + "echo \"[systemtests] Building tutorial images...\"\n" + "docker compose --file docker-compose.tutorial.yaml build\n" + "\n" + "echo \"[systemtests] Running tutorial containers...\"\n" + "docker compose --file docker-compose.tutorial.yaml up\n" + "\n" + "if [ -f docker-compose.field_compare.yaml ]; then\n" + " echo \"[systemtests] Running fieldcompare...\"\n" + " docker compose --file docker-compose.field_compare.yaml up --exit-code-from field-compare\n" + "fi\n" + ) + # Make the script executable for convenience; even if this bit + # does not survive archiving, users can still run it via + # `sh rerun_systemtest.sh`. + try: + rerun_script_path.chmod(rerun_script_path.stat().st_mode | 0o111) + except Exception: + logging.debug( + f"Could not mark {rerun_script_path} as executable; continuing anyway.") + stdout_data = [] stderr_data = [] From 859230521b5b4d50e935f60b9dea13d3cbdf1347 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Thu, 25 Jun 2026 08:50:40 +0530 Subject: [PATCH 2/2] Address review feedback for portable system test artifact replay. Rename rerun script, write fieldcompare compose at build time, fix replay permissions, and update docs/changelog for #724. --- changelog-entries/724.md | 2 +- tools/tests/README.md | 12 +++++++----- .../{rerun_systemtest.sh => rerun-system-test.sh} | 6 ++++++ tools/tests/systemtests/Systemtest.py | 14 +++++++++----- 4 files changed, 23 insertions(+), 11 deletions(-) rename tools/tests/{rerun_systemtest.sh => rerun-system-test.sh} (59%) diff --git a/changelog-entries/724.md b/changelog-entries/724.md index df4a54cbb..1199ecfd3 100644 --- a/changelog-entries/724.md +++ b/changelog-entries/724.md @@ -1 +1 @@ -- System test run directories use relative Docker Compose paths and include a `rerun_systemtest.sh` script so CI artifacts can be downloaded and replayed locally (Closes #387). +- Added the ability to replay system tests: System test artifacts now include a `rerun-system-test.sh` script and are prepared to be portable. [#724](https://github.com/precice/tutorials/pull/724) diff --git a/tools/tests/README.md b/tools/tests/README.md index cbb9d883c..ed90ea53e 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -123,8 +123,8 @@ runs/ ├── tools/ # Dockerfiles and helpers (shared) └── __/ # one folder per system test ├── docker-compose.tutorial.yaml - ├── docker-compose.field_compare.yaml # if fieldcompare ran - ├── rerun_systemtest.sh + ├── docker-compose.field_compare.yaml # written at build time when compare is configured + ├── rerun-system-test.sh ├── system-tests-build.log ├── system-tests-run.log ├── system-tests-compare.log @@ -135,11 +135,13 @@ To re-run one test locally: 1. Extract the zip and keep the `runs/` layout (the test folder needs the sibling `tools/` directory). 2. `cd` into the test folder. -3. Run `./rerun_systemtest.sh` (or `sh rerun_systemtest.sh`). +3. Run `./rerun-system-test.sh` (or `sh rerun-system-test.sh`). The script rebuilds images, runs the tutorial, and (if present) runs fieldcompare with `--exit-code-from field-compare`, matching the Python runner. Compose paths are relative to the test folder (`..` is the parent `runs/` directory), so you can move the extracted tree elsewhere on a Linux host with Docker. -Fieldcompare requires reference results in the artifact (unpacked during the original CI run) or you must unpack them manually first. +`docker-compose.field_compare.yaml` is written when the test is prepared for Docker build. If an older artifact omits it, the original run likely failed before compare. The replay script fixes common permission issues from extracted archives (`chmod` before compose). + +Fieldcompare requires reference results in the artifact. If not already unpacked during the original CI run, unpack them manually first. ## Extending @@ -237,7 +239,7 @@ User-facing tools: - `print_case_combinations.py`: Prints all possible combinations of tutorial cases, using the `metadata.yaml` files. - `build_docker_images.py`: Build the Docker images for each test - `generate_reference_results.py`: Executes the system tests with the versions defined in `reference_versions.yaml` and generates the reference data archives, with the names described in `tests.yaml`. (should only be used by the CI Pipeline) - - `rerun_systemtest.sh`: Helper script copied into each run directory so CI artifacts can be replayed locally (see [#387](https://github.com/precice/tutorials/issues/387)). + - `rerun-system-test.sh`: Helper script copied into each run directory so CI artifacts can be replayed locally. Implementation scripts: diff --git a/tools/tests/rerun_systemtest.sh b/tools/tests/rerun-system-test.sh similarity index 59% rename from tools/tests/rerun_systemtest.sh rename to tools/tests/rerun-system-test.sh index 66210e2ab..97f6fd54e 100644 --- a/tools/tests/rerun_systemtest.sh +++ b/tools/tests/rerun-system-test.sh @@ -3,6 +3,10 @@ set -e -u cd "$(dirname "$0")" +# Unzipped CI artifacts are often owned by the host user. The prepare container +# runs as precice and must edit precice-config.xml in this directory. +chmod -R a+rwX . + echo "[systemtests] Building tutorial images..." docker compose --file docker-compose.tutorial.yaml build @@ -12,4 +16,6 @@ docker compose --file docker-compose.tutorial.yaml up if [ -f docker-compose.field_compare.yaml ]; then echo "[systemtests] Running fieldcompare..." docker compose --file docker-compose.field_compare.yaml up --exit-code-from field-compare +else + echo "[systemtests] Skipping fieldcompare (docker-compose.field_compare.yaml not present; the original run likely failed before compare)." fi diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 8c269f66b..190ab2371 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -737,13 +737,13 @@ def __archive_fieldcompare_diffs(self) -> None: self, ) - def __copy_rerun_systemtest_script(self) -> None: - """Copy tools/tests/rerun_systemtest.sh into the run directory for artifact replay.""" - rerun_src = PRECICE_TESTS_DIR / "rerun_systemtest.sh" + def __copy_rerun_system_test_script(self) -> None: + """Copy tools/tests/rerun-system-test.sh into the run directory for artifact replay.""" + rerun_src = PRECICE_TESTS_DIR / "rerun-system-test.sh" if not rerun_src.is_file(): raise FileNotFoundError( f"Missing {rerun_src}. It is required for portable CI artifact replay.") - rerun_dst = self.system_test_dir / "rerun_systemtest.sh" + rerun_dst = self.system_test_dir / "rerun-system-test.sh" shutil.copy2(rerun_src, rerun_dst) try: rerun_dst.chmod(rerun_dst.stat().st_mode | 0o111) @@ -762,7 +762,11 @@ def _build_docker(self): with open(docker_compose_path, 'w') as file: file.write(docker_compose_content) - self.__copy_rerun_systemtest_script() + field_compare_compose_path = self.system_test_dir / "docker-compose.field_compare.yaml" + with open(field_compare_compose_path, 'w') as file: + file.write(self.__get_field_compare_compose_file()) + + self.__copy_rerun_system_test_script() exit_code, stdout_data, stderr_data = self._run_docker_compose_subprocess( [