From ad4361454d0f421a6590e9cc64ec13ea25c355b3 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 19 Aug 2026 18:55:30 +1000 Subject: [PATCH 1/3] Run the whole parallel directory, and run it at four ranks as well as two MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two scripts, two different holes. scripts/test.sh (what CI runs) used tests/parallel/test_075*py and test_10*py; scripts/test_levels.sh (what ./uw test runs) used tests/parallel/test_07*py. Of the 32 collectible files in that directory, test.sh named 14 and missed test_0005, test_0700, test_0760..test_0790, test_0855 and test_0873; test_levels.sh missed test_0005, test_0855, test_0873 and the whole test_10* solver set. Three files therefore ran at NO rank count in either script: test_0005, test_0855, and test_0873 — the last added two days ago in #596 to guard against a parallel hang. Both now name the directory. A glob that names ranges grows holes as files are added between them, which is the #570 class and is how #611 survived unnoticed. Both scripts also run the set at four ranks. Two ranks is a special case: the defect this suite exists to catch is a collective entered by some ranks and not others, and with two the mismatched pair often still meets. Every instance found recently passed at np=2 and hung at np=4 — the conditional collective in #609, and #611 itself. test_levels.sh already had --full-parallel for this and was pointing it at the narrower glob. Measured on this directory, machine otherwise idle: np=2 128 passed, 11 skipped 156 s np=4 135 passed, 3 skipped 170 s (1 deselected) end to end, scripts/test.sh --p 2: 135 passed, 3 skipped, 1 deselected, 198 s for the parallel section, rc=0. The skip counts differ because some tests require four ranks and skip at two, which is a second reason to run both. The single deselection is #611: test_global_evaluate_after_migration passes at np=2 and hangs at np=4 on development. Its node id carries no `tests/` prefix because tests/pytest.ini puts rootdir at `tests/`; a deselect that does not match is ignored in silence, which cost two wrong diagnoses while measuring this. Underworld development team with AI support from Claude Code --- scripts/test.sh | 45 ++++++++++++++++++++++++++++++++---------- scripts/test_levels.sh | 19 +++++++++++++++--- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 93a69edf..69a33b70 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -148,16 +148,41 @@ if [ $PARALLEL_RANKS -gt 0 ]; then # - Solver operations # - Global evaluations - echo "Testing global statistics and parallel operations..." - mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_075*py || status=1 - - # Parallel SOLVER tests. This line was commented out, so test_1017 and - # test_1062..test_1069 — the whole rotated / constrained / MG parallel set, - # including the partition-independence guards for the rotated nodal normal - # (#560) and the mesh boundary normal (#564) — executed at NO rank count - # in CI. - echo "Testing parallel solvers..." - mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_10*py || status=1 + # The WHOLE directory, not a set of globs. It used to be `test_075*py` and + # `test_10*py`, which between them named 14 of the 32 collectible files and + # left a hole from 0760 to 0999 — so test_0760, test_0765..test_0790, + # test_0855 and test_0873 ran in parallel at NO rank count, in CI or + # locally. That is how the np=4 hang in #611 survived unnoticed, and it is + # the #570 class: a glob that names ranges will grow holes as files are + # added between them. + # + # Measured on this directory at np=2: 128 passed, 11 skipped, 156 s. + echo "Testing parallel operations, swarms and solvers..." + mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/ || status=1 + + # A SECOND pass at four ranks, because two is a special case. The failure + # this suite exists to catch is a collective entered by some ranks and not + # others, and with two ranks the mismatched pair often still meets — the + # skipped branch does not arise on a two-way partition, or one rank's + # internal barrier is satisfied by the other's next collective. Every such + # defect found recently passed at np=2 and hung at np=4: the conditional + # collective in #609, and #611. + # + # Measured on this directory: np=2 128 passed / 11 skipped / 156 s; np=4 + # 135 passed / 3 skipped / 170 s. The skip counts differ because some tests + # require four ranks and are skipped at two — which is the other reason to + # run both. + # + # The one deselection is #611: test_global_evaluate_after_migration passes + # at np=2 and hangs at np=4 on development. The node id has no `tests/` + # prefix because tests/pytest.ini puts rootdir at `tests/`, and a deselect + # that does not match is ignored in silence. + if [ "$PARALLEL_RANKS" -ne 4 ]; then + echo "Testing the same set at 4 ranks (np=2 is a special case)..." + mpirun -n 4 python -m pytest --with-mpi tests/parallel/ \ + --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \ + || status=1 + fi # echo "Testing parallel I/O..." # mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_io*py || status=1 diff --git a/scripts/test_levels.sh b/scripts/test_levels.sh index 2f3888d5..f0f52cf4 100755 --- a/scripts/test_levels.sh +++ b/scripts/test_levels.sh @@ -212,9 +212,16 @@ run_level_2() { echo "Running parallel tests (MPI)" echo "==========================================" - # Parallel tests with specified number of ranks + # The WHOLE directory, not `test_07*py`. That glob covered + # test_0700..test_0790 and so missed test_0005, test_0855, + # test_0873 and the entire test_10* solver set. `scripts/test.sh` + # had a different hole (test_075* and test_10*, missing + # test_0760..test_0790), so between the two scripts three files ran + # at no rank count anywhere: test_0005, test_0855 and test_0873. + # A glob naming ranges grows holes as files are added between them + # (#570, #611). echo "Testing with $PARALLEL_RANKS MPI ranks..." - if mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_07*py $VERBOSE; then + if mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/ $VERBOSE; then echo "✅ PASSED: Parallel tests ($PARALLEL_RANKS ranks)" else echo "❌ FAILED: Parallel tests ($PARALLEL_RANKS ranks)" @@ -224,8 +231,14 @@ run_level_2() { # Optional: Test with 4 ranks if --full-parallel specified if [ $FULL_PARALLEL -eq 1 ]; then echo "" + # The deselection is #611: this test passes at np=2 and hangs + # at np=4 on development. The node id carries no `tests/` + # prefix because tests/pytest.ini puts rootdir at `tests/`, and + # a deselect that does not match is ignored in silence. echo "Running extended parallel tests (4 ranks)..." - if mpirun -n 4 python -m pytest --with-mpi tests/parallel/test_07*py $VERBOSE; then + if mpirun -n 4 python -m pytest --with-mpi tests/parallel/ \ + --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \ + $VERBOSE; then echo "✅ PASSED: Parallel tests (4 ranks)" else echo "❌ FAILED: Parallel tests (4 ranks)" From 4a532bbc804ea995431a97bc4f9125d3f23efd0f Mon Sep 17 00:00:00 2001 From: lmoresi Date: Thu, 20 Aug 2026 10:43:40 +1000 Subject: [PATCH 2/3] Make the four-rank pass opt-in: it does not fit the CI budget Run unconditionally, the second parallel pass took the test job past the 120-minute cap. Recent runs on this repo take 26 to 40 minutes, so there was about 80 minutes of headroom; on a two-core runner np=4 is oversubscribed and the pass costs far more than the ~3 minutes it takes on a workstation. It is now behind --full-parallel, off by default, which is the same shape test_levels.sh already used for its own four-rank pass. CI keeps the part that matters and costs little: the whole tests/parallel/ directory at the requested rank count, which is what closes the glob hole. Verified: ./scripts/test.sh --p 2 --parallel-only exits 0 and runs no four-rank pass. Underworld development team with AI support from Claude Code --- scripts/test.sh | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 69a33b70..175d1cdd 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,11 +4,13 @@ # # Usage: ./test.sh [OPTIONS] # --p N Run parallel tests with N MPI ranks (default: skip parallel tests) +# --full-parallel Add a second parallel pass at 4 ranks (see below; slow on CI) # --parallel-only Run ONLY parallel tests (skip all serial tests) # # Examples: # ./test.sh # All serial tests only # ./test.sh --p 2 # All serial + parallel (2 ranks) +# ./test.sh --p 2 --full-parallel # ... and again at 4 ranks # ./test.sh --parallel-only --p 2 # Only parallel tests (debugging) # # We do not run one monolithic pytest because tests produce a large number of @@ -20,19 +22,27 @@ status=0 # Parse arguments PARALLEL_RANKS=0 PARALLEL_ONLY=0 +# Second parallel pass at four ranks. OFF by default: on a 2-core CI runner +# np=4 is oversubscribed and the pass costs far more than the ~3 minutes it +# takes on a workstation — enough to exceed the 120-minute job cap (#573). +FULL_PARALLEL=0 while [[ $# -gt 0 ]]; do case $1 in --p) PARALLEL_RANKS="$2" shift 2 ;; + --full-parallel) + FULL_PARALLEL=1 + shift + ;; --parallel-only) PARALLEL_ONLY=1 shift ;; *) echo "Unknown option: $1" - echo "Usage: $0 [--p N] [--parallel-only]" + echo "Usage: $0 [--p N] [--full-parallel] [--parallel-only]" exit 1 ;; esac @@ -168,16 +178,21 @@ if [ $PARALLEL_RANKS -gt 0 ]; then # defect found recently passed at np=2 and hung at np=4: the conditional # collective in #609, and #611. # - # Measured on this directory: np=2 128 passed / 11 skipped / 156 s; np=4 + # Measured on a workstation: np=2 128 passed / 11 skipped / 156 s; np=4 # 135 passed / 3 skipped / 170 s. The skip counts differ because some tests # require four ranks and are skipped at two — which is the other reason to # run both. # + # Opt-in via --full-parallel. A CI runner has two cores, so np=4 is + # oversubscribed there and this pass costs far more than it does locally: + # run unconditionally it took the test job past the 120-minute cap. It + # belongs in a nightly or a separate job (#573) rather than on every PR. + # # The one deselection is #611: test_global_evaluate_after_migration passes # at np=2 and hangs at np=4 on development. The node id has no `tests/` # prefix because tests/pytest.ini puts rootdir at `tests/`, and a deselect # that does not match is ignored in silence. - if [ "$PARALLEL_RANKS" -ne 4 ]; then + if [ $FULL_PARALLEL -eq 1 ] && [ "$PARALLEL_RANKS" -ne 4 ]; then echo "Testing the same set at 4 ranks (np=2 is a special case)..." mpirun -n 4 python -m pytest --with-mpi tests/parallel/ \ --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \ From 778e9108f4cf3406f7c55834a799e6810a0632ce Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 21 Aug 2026 09:43:45 +1000 Subject: [PATCH 3/3] Cover tests/parallel by enumeration, and keep the batching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two requirements that pull against each other, and the previous commit met only one. Coverage: the globs `test_075*py` and `test_10*py` named 14 of the 32 files and left a hole from 0760 to 0999, so test_0760, test_0765..test_0790, test_0855 and test_0873 ran in parallel at no rank count. The list is now enumerated from the directory, so it covers by construction rather than by ranges that grow holes as files are added between them. Batching: this script does not run one monolithic pytest, for the reason in its own header — PETSc objects accumulate across files and tests begin to interact. Handing the whole directory to a single mpirun took the CI test job past its 120-minute cap twice. The enumeration is therefore chunked, PARALLEL_BATCH files at a time, default 6. Measured in-environment at np=2: 32 files, 128 passed, 180 s across 6 batches. The four-rank pass stays opt-in behind --full-parallel, where "1 deselected" confirms the #611 exclusion matches. Underworld development team with AI support from Claude Code --- scripts/test.sh | 74 ++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 175d1cdd..69c804ea 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -158,50 +158,54 @@ if [ $PARALLEL_RANKS -gt 0 ]; then # - Solver operations # - Global evaluations - # The WHOLE directory, not a set of globs. It used to be `test_075*py` and - # `test_10*py`, which between them named 14 of the 32 collectible files and - # left a hole from 0760 to 0999 — so test_0760, test_0765..test_0790, - # test_0855 and test_0873 ran in parallel at NO rank count, in CI or - # locally. That is how the np=4 hang in #611 survived unnoticed, and it is - # the #570 class: a glob that names ranges will grow holes as files are - # added between them. + # Every file under tests/parallel/, in BATCHES. Two things matter here and + # they pull in opposite directions. # - # Measured on this directory at np=2: 128 passed, 11 skipped, 156 s. - echo "Testing parallel operations, swarms and solvers..." - mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/ || status=1 - - # A SECOND pass at four ranks, because two is a special case. The failure - # this suite exists to catch is a collective entered by some ranks and not - # others, and with two ranks the mismatched pair often still meets — the - # skipped branch does not arise on a two-way partition, or one rank's - # internal barrier is satisfied by the other's next collective. Every such - # defect found recently passed at np=2 and hung at np=4: the conditional - # collective in #609, and #611. + # Coverage: the batches used to be `test_075*py` and `test_10*py`, which + # between them named 14 of the 32 files and left a hole from 0760 to 0999 — + # test_0760, test_0765..test_0790, test_0855 and test_0873 ran in parallel + # at NO rank count, in CI or locally. That is how the np=4 hang in #611 + # survived unnoticed, and it is the #570 class: a glob naming ranges grows + # holes as files are added between them. So the list is ENUMERATED, and + # covers the directory by construction. # - # Measured on a workstation: np=2 128 passed / 11 skipped / 156 s; np=4 - # 135 passed / 3 skipped / 170 s. The skip counts differ because some tests - # require four ranks and are skipped at two — which is the other reason to - # run both. + # Batching: this script does not run one monolithic pytest, for the reason + # in its header — PETSc objects accumulate across files and tests start + # interacting. Handing the whole directory to a single mpirun took the CI + # job past its 120-minute cap while the same set in batches costs a couple + # of minutes. So the enumeration is chunked rather than passed in one go. + PARALLEL_BATCH=${PARALLEL_BATCH:-6} + PARALLEL_FILES=(tests/parallel/test_*.py) + echo "Testing parallel operations, swarms and solvers" + echo " ${#PARALLEL_FILES[@]} files in batches of $PARALLEL_BATCH" + for ((i = 0; i < ${#PARALLEL_FILES[@]}; i += PARALLEL_BATCH)); do + mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi \ + "${PARALLEL_FILES[@]:i:PARALLEL_BATCH}" || status=1 + done + + # A SECOND pass at four ranks, because two is a special case: the failure + # this suite exists to catch is a collective entered by some ranks and not + # others, and with two ranks the mismatched pair often still meets. Both + # recent instances passed at np=2 and hung at np=4 (#609's conditional + # collective, and #611). # # Opt-in via --full-parallel. A CI runner has two cores, so np=4 is - # oversubscribed there and this pass costs far more than it does locally: - # run unconditionally it took the test job past the 120-minute cap. It - # belongs in a nightly or a separate job (#573) rather than on every PR. + # oversubscribed there; this belongs in a nightly or a separate job (#573). # - # The one deselection is #611: test_global_evaluate_after_migration passes - # at np=2 and hangs at np=4 on development. The node id has no `tests/` - # prefix because tests/pytest.ini puts rootdir at `tests/`, and a deselect - # that does not match is ignored in silence. + # The deselection is #611: that test passes at np=2 and hangs at np=4 on + # development. The node id has no `tests/` prefix because tests/pytest.ini + # puts rootdir at `tests/`, and a deselect that does not match is ignored in + # silence — confirm "1 deselected" in the output when changing it. if [ $FULL_PARALLEL -eq 1 ] && [ "$PARALLEL_RANKS" -ne 4 ]; then echo "Testing the same set at 4 ranks (np=2 is a special case)..." - mpirun -n 4 python -m pytest --with-mpi tests/parallel/ \ - --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \ - || status=1 + for ((i = 0; i < ${#PARALLEL_FILES[@]}; i += PARALLEL_BATCH)); do + mpirun -n 4 python -m pytest --with-mpi \ + "${PARALLEL_FILES[@]:i:PARALLEL_BATCH}" \ + --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \ + || status=1 + done fi - # echo "Testing parallel I/O..." - # mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_io*py || status=1 - echo "Parallel tests complete" echo "==========================================" else