From f96316aa59ae498e10ca0789a64e935674647c17 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Sun, 23 Aug 2026 17:58:14 +1000 Subject: [PATCH 1/2] Blame the right rank, rather than require every rank to have dumped test_0054's end-to-end check asserted the waiting group was exactly [0, 2, 3]. On an oversubscribed CI runner a rank can be scheduled too little to dump inside the window, and the group came back as [0] -- a true report of a slower machine, not a defect. It went red on development after #630 merged. What the tool has to get right is which rank is BLAMED, so that is what is asserted now: the majority group sits at the collective, rank 1 is not in it, the group contains only ranks that entered it, and rank 1 is named as the odd one out. Requiring a particular number of witnesses tested the scheduler. Underworld development team with AI support from Claude Code --- tests/test_0054_hang_report.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/test_0054_hang_report.py b/tests/test_0054_hang_report.py index 66bd625f..8586ac0e 100644 --- a/tests/test_0054_hang_report.py +++ b/tests/test_0054_hang_report.py @@ -235,15 +235,28 @@ def test_end_to_end_names_the_divergent_rank(tmp_path): biggest_where, biggest_ranks, _stack = groups[0] report = hang_report.format_report(states) - assert biggest_ranks == [0, 2, 3], ( - f"the waiting ranks were {biggest_ranks}, not [0, 2, 3]:\n{report}" - ) + # NOT `biggest_ranks == [0, 2, 3]`. On an oversubscribed runner a rank can + # be scheduled too little to dump inside the window, and requiring all three + # made this fail on CI with the waiting group [0] -- a true report of a + # slower machine, not a defect. What the tool must get right is WHICH RANK + # IS BLAMED, so that is what is asserted. assert biggest_where[2] == "reduce_the_count", ( - f"the majority was located at {biggest_where}, not at the collective" + f"the majority was located at {biggest_where}, not at the collective:\n" + f"{report}" + ) + assert 1 not in biggest_ranks, ( + f"rank 1 branched around the collective and must not be in the waiting " + f"group {biggest_ranks}:\n{report}" ) + assert set(biggest_ranks) <= {0, 2, 3} and biggest_ranks, ( + f"the waiting group {biggest_ranks} contains a rank that never entered " + f"the collective:\n{report}" + ) + odd_ones_out = sorted(r for _w, ranks, _s in groups[1:] for r in ranks) + moving - assert odd_ones_out == [1], ( - f"rank 1 took the branch; the report blamed {odd_ones_out}:\n{report}" + assert 1 in odd_ones_out, ( + f"rank 1 took the branch and must be named as the odd one out; the " + f"report blamed {odd_ones_out}:\n{report}" ) # The verdict has to point at the branch, not merely list stacks. assert "where the bug is" in report From 317a9e2db50ecf3c63526732783123e61e2cfbc8 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Sun, 23 Aug 2026 20:17:46 +1000 Subject: [PATCH 2/2] Give the watchdog room past import before judging where the ranks are At a 1.0 s watchdog and a 25 s window the end-to-end check passed here and failed on CI with the majority located in `importlib._bootstrap`: four oversubscribed ranks take longer to `import underworld3` than the watchdog allowed, so the dump file filled with import frames and the job was killed before the collective produced enough dumps to outvote them. The timings were tuned to a workstation. 5 s watchdog, 75 s window. The assertion also now names the frame the ranks were actually found in -- "not at the collective" without saying where cost a CI round trip to diagnose. Underworld development team with AI support from Claude Code --- tests/test_0054_hang_report.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_0054_hang_report.py b/tests/test_0054_hang_report.py index 8586ac0e..8220e645 100644 --- a/tests/test_0054_hang_report.py +++ b/tests/test_0054_hang_report.py @@ -194,7 +194,7 @@ def reduce_the_count(undecided): """ -@pytest.mark.timeout(300) +@pytest.mark.timeout(600) @pytest.mark.skipif(shutil.which("mpirun") is None, reason="needs mpirun") def test_end_to_end_names_the_divergent_rank(tmp_path): """A real four-rank job that really hangs, killed, then analysed. @@ -211,14 +211,20 @@ def test_end_to_end_names_the_divergent_rank(tmp_path): script.write_text(textwrap.dedent(DIVERGENT)) dumps = tmp_path / "uw-hang-dumps" + # The watchdog must be longer than a plausible `import underworld3` and the + # window long enough for the blocked phase to dominate. At 1.0 s / 25 s this + # passed here and failed on CI with the majority located in + # `importlib._bootstrap`: four oversubscribed ranks take longer to import + # than the watchdog allowed, so the file filled with import dumps and the + # job was killed before the collective produced enough to outvote them. environment = dict( os.environ, - UW_HANG_WATCHDOG="1.0", + UW_HANG_WATCHDOG="5.0", UW_HANG_WATCHDOG_DIR=str(dumps), UW_NO_USAGE_METRICS="1", ) stderr = _run_until_it_hangs( - [sys.executable, "-u", str(script)], ranks=4, seconds=25, + [sys.executable, "-u", str(script)], ranks=4, seconds=75, environment=environment, ) @@ -241,8 +247,9 @@ def test_end_to_end_names_the_divergent_rank(tmp_path): # slower machine, not a defect. What the tool must get right is WHICH RANK # IS BLAMED, so that is what is asserted. assert biggest_where[2] == "reduce_the_count", ( - f"the majority was located at {biggest_where}, not at the collective:\n" - f"{report}" + f"the majority was located at {biggest_where[0]}:{biggest_where[1]} in " + f"{biggest_where[2]}, not at the collective. If that is an import frame " + f"the watchdog fired before the ranks got there.\n{report}" ) assert 1 not in biggest_ranks, ( f"rank 1 branched around the collective and must not be in the waiting "