diff --git a/scripts/test.sh b/scripts/test.sh index 93a69edf..69c804ea 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,11 +4,13 @@ # # Usage: ./test.sh [OPTIONS] # --p N Run parallel tests with N MPI ranks (default: skip parallel tests) +# --full-parallel Add a second parallel pass at 4 ranks (see below; slow on CI) # --parallel-only Run ONLY parallel tests (skip all serial tests) # # Examples: # ./test.sh # All serial tests only # ./test.sh --p 2 # All serial + parallel (2 ranks) +# ./test.sh --p 2 --full-parallel # ... and again at 4 ranks # ./test.sh --parallel-only --p 2 # Only parallel tests (debugging) # # We do not run one monolithic pytest because tests produce a large number of @@ -20,19 +22,27 @@ status=0 # Parse arguments PARALLEL_RANKS=0 PARALLEL_ONLY=0 +# Second parallel pass at four ranks. OFF by default: on a 2-core CI runner +# np=4 is oversubscribed and the pass costs far more than the ~3 minutes it +# takes on a workstation — enough to exceed the 120-minute job cap (#573). +FULL_PARALLEL=0 while [[ $# -gt 0 ]]; do case $1 in --p) PARALLEL_RANKS="$2" shift 2 ;; + --full-parallel) + FULL_PARALLEL=1 + shift + ;; --parallel-only) PARALLEL_ONLY=1 shift ;; *) echo "Unknown option: $1" - echo "Usage: $0 [--p N] [--parallel-only]" + echo "Usage: $0 [--p N] [--full-parallel] [--parallel-only]" exit 1 ;; esac @@ -148,19 +158,53 @@ if [ $PARALLEL_RANKS -gt 0 ]; then # - Solver operations # - Global evaluations - echo "Testing global statistics and parallel operations..." - mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_075*py || status=1 - - # Parallel SOLVER tests. This line was commented out, so test_1017 and - # test_1062..test_1069 — the whole rotated / constrained / MG parallel set, - # including the partition-independence guards for the rotated nodal normal - # (#560) and the mesh boundary normal (#564) — executed at NO rank count - # in CI. - echo "Testing parallel solvers..." - mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_10*py || status=1 - - # echo "Testing parallel I/O..." - # mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_io*py || status=1 + # Every file under tests/parallel/, in BATCHES. Two things matter here and + # they pull in opposite directions. + # + # Coverage: the batches used to be `test_075*py` and `test_10*py`, which + # between them named 14 of the 32 files and left a hole from 0760 to 0999 — + # test_0760, test_0765..test_0790, test_0855 and test_0873 ran in parallel + # at NO rank count, in CI or locally. That is how the np=4 hang in #611 + # survived unnoticed, and it is the #570 class: a glob naming ranges grows + # holes as files are added between them. So the list is ENUMERATED, and + # covers the directory by construction. + # + # Batching: this script does not run one monolithic pytest, for the reason + # in its header — PETSc objects accumulate across files and tests start + # interacting. Handing the whole directory to a single mpirun took the CI + # job past its 120-minute cap while the same set in batches costs a couple + # of minutes. So the enumeration is chunked rather than passed in one go. + PARALLEL_BATCH=${PARALLEL_BATCH:-6} + PARALLEL_FILES=(tests/parallel/test_*.py) + echo "Testing parallel operations, swarms and solvers" + echo " ${#PARALLEL_FILES[@]} files in batches of $PARALLEL_BATCH" + for ((i = 0; i < ${#PARALLEL_FILES[@]}; i += PARALLEL_BATCH)); do + mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi \ + "${PARALLEL_FILES[@]:i:PARALLEL_BATCH}" || status=1 + done + + # A SECOND pass at four ranks, because two is a special case: the failure + # this suite exists to catch is a collective entered by some ranks and not + # others, and with two ranks the mismatched pair often still meets. Both + # recent instances passed at np=2 and hung at np=4 (#609's conditional + # collective, and #611). + # + # Opt-in via --full-parallel. A CI runner has two cores, so np=4 is + # oversubscribed there; this belongs in a nightly or a separate job (#573). + # + # The deselection is #611: that test passes at np=2 and hangs at np=4 on + # development. The node id has no `tests/` prefix because tests/pytest.ini + # puts rootdir at `tests/`, and a deselect that does not match is ignored in + # silence — confirm "1 deselected" in the output when changing it. + if [ $FULL_PARALLEL -eq 1 ] && [ "$PARALLEL_RANKS" -ne 4 ]; then + echo "Testing the same set at 4 ranks (np=2 is a special case)..." + for ((i = 0; i < ${#PARALLEL_FILES[@]}; i += PARALLEL_BATCH)); do + mpirun -n 4 python -m pytest --with-mpi \ + "${PARALLEL_FILES[@]:i:PARALLEL_BATCH}" \ + --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \ + || status=1 + done + fi echo "Parallel tests complete" echo "==========================================" diff --git a/scripts/test_levels.sh b/scripts/test_levels.sh index 2f3888d5..f0f52cf4 100755 --- a/scripts/test_levels.sh +++ b/scripts/test_levels.sh @@ -212,9 +212,16 @@ run_level_2() { echo "Running parallel tests (MPI)" echo "==========================================" - # Parallel tests with specified number of ranks + # The WHOLE directory, not `test_07*py`. That glob covered + # test_0700..test_0790 and so missed test_0005, test_0855, + # test_0873 and the entire test_10* solver set. `scripts/test.sh` + # had a different hole (test_075* and test_10*, missing + # test_0760..test_0790), so between the two scripts three files ran + # at no rank count anywhere: test_0005, test_0855 and test_0873. + # A glob naming ranges grows holes as files are added between them + # (#570, #611). echo "Testing with $PARALLEL_RANKS MPI ranks..." - if mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_07*py $VERBOSE; then + if mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/ $VERBOSE; then echo "✅ PASSED: Parallel tests ($PARALLEL_RANKS ranks)" else echo "❌ FAILED: Parallel tests ($PARALLEL_RANKS ranks)" @@ -224,8 +231,14 @@ run_level_2() { # Optional: Test with 4 ranks if --full-parallel specified if [ $FULL_PARALLEL -eq 1 ]; then echo "" + # The deselection is #611: this test passes at np=2 and hangs + # at np=4 on development. The node id carries no `tests/` + # prefix because tests/pytest.ini puts rootdir at `tests/`, and + # a deselect that does not match is ignored in silence. echo "Running extended parallel tests (4 ranks)..." - if mpirun -n 4 python -m pytest --with-mpi tests/parallel/test_07*py $VERBOSE; then + if mpirun -n 4 python -m pytest --with-mpi tests/parallel/ \ + --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \ + $VERBOSE; then echo "✅ PASSED: Parallel tests (4 ranks)" else echo "❌ FAILED: Parallel tests (4 ranks)"