From c43d05ec7849fbd25867b9cfcc75d32d9f140342 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Tue, 2 Jun 2026 23:40:42 -0600 Subject: [PATCH 01/11] feat: add --max-suite-retries option to cap total reruns across suite - New CLI option `--max-suite-retries` (int, default None = no limit); once the suite-wide rerun count hits the cap, failing tests are logged as final failures without further retry - `StatusDB` gains thread-safe `increment_suite_reruns()` / `get_suite_reruns()` via `threading.Lock` for the single-process path - `ServerStatusDB` overrides use the existing `rerunfailures_db` dict with a `"__suite__"` key and an atomic socket `inc` operation in `run_connection` - `ClientStatusDB` overrides route increment through the socket to the master - Five new tests covering: cap enforcement, pass-through when under cap, zero disables all reruns, passing tests don't consume the budget, and standalone option no-op - CHANGES.rst and README.rst updated Fixes #298 --- Co-authored-by: Claude Code --- CHANGES.rst | 8 +++- README.rst | 15 +++++++ src/pytest_rerunfailures.py | 54 ++++++++++++++++++++++ tests/test_pytest_rerunfailures.py | 72 ++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7960238..4128c23 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,13 @@ Changelog 16.4 (unreleased) ----------------- -- Nothing changed yet. +Features +++++++++ + +- Add ``--max-suite-retries`` option to cap the total number of reruns across + the entire test suite. Once the limit is reached, no further reruns occur + regardless of per-test ``--reruns`` or ``@pytest.mark.flaky`` settings. + Fixes `#298 `_. 16.3 (2026-05-22) diff --git a/README.rst b/README.rst index 30d2669..d8d9a6a 100644 --- a/README.rst +++ b/README.rst @@ -220,6 +220,21 @@ setting. To make them additive instead, pass ``--reruns-mode=append``. With $ pytest --reruns 4 --reruns-mode append +Limit total reruns across the suite +------------------------------------ + +To cap the total number of reruns across the entire test suite regardless of +how many individual tests fail, pass ``--max-suite-retries``. Once the limit +is reached, no further reruns occur even if individual tests have remaining +retries: + +.. code-block:: bash + + $ pytest --reruns 3 --max-suite-retries 10 + +This is useful in large test suites to bound resource usage when many tests +are flaky at the same time. + Show tracebacks for retried failures ------------------------------------ diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 127cdc9..366f6bd 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -120,6 +120,15 @@ def pytest_addoption(parser): "'rerun test summary info' section, which is emitted automatically " "when this flag is set.", ) + group._addoption( + "--max-suite-retries", + action="store", + dest="max_suite_retries", + type=int, + default=None, + help="Maximum total number of reruns across the entire test suite. " + "Once this limit is reached, no further reruns will occur.", + ) arg_type = "string" parser.addini("reruns", RERUNS_DESC, type=arg_type) @@ -430,6 +439,18 @@ class StatusDB: def __init__(self): self.delim = b"\n" self.hmap = {} + self._suite_rerun_count = 0 + self._suite_lock = threading.Lock() + + def increment_suite_reruns(self) -> int: + """Atomically increment the suite-wide rerun counter; return new total.""" + with self._suite_lock: + self._suite_rerun_count += 1 + return self._suite_rerun_count + + def get_suite_reruns(self) -> int: + """Return the current suite-wide rerun count.""" + return self._suite_rerun_count def _hash(self, crashitem: str) -> str: if crashitem not in self.hmap: @@ -514,6 +535,11 @@ def run_connection(self, conn): self._set(i, k, int(v)) elif op == "get": self._sock_send(conn, str(self._get(i, k))) + elif op == "inc": + with self._suite_lock: + new_v = self._get(i, k) + 1 + self._set(i, k, new_v) + self._sock_send(conn, str(new_v)) def _set(self, i: str, k: str, v: int): if i not in self.rerunfailures_db: @@ -526,6 +552,17 @@ def _get(self, i: str, k: str) -> int: except KeyError: return 0 + def increment_suite_reruns(self) -> int: + """Atomically increment the suite-wide rerun counter; return new total.""" + with self._suite_lock: + new_v = self._get("__suite__", "r") + 1 + self._set("__suite__", "r", new_v) + return new_v + + def get_suite_reruns(self) -> int: + """Return the current suite-wide rerun count.""" + return self._get("__suite__", "r") + class ClientStatusDB(SocketDB): def __init__(self, sock_port): @@ -539,6 +576,15 @@ def _get(self, i: str, k: str) -> int: self._sock_send(self.sock, "|".join(("get", i, k, ""))) return int(self._sock_recv(self.sock)) + def increment_suite_reruns(self) -> int: + """Atomically increment the suite-wide rerun counter; return new total.""" + self._sock_send(self.sock, "|".join(("inc", "__suite__", "r", ""))) + return int(self._sock_recv(self.sock)) + + def get_suite_reruns(self) -> int: + """Return the current suite-wide rerun count.""" + return self._get("__suite__", "r") + suspended_finalizers = {} @@ -638,6 +684,14 @@ def pytest_runtest_protocol(item, nextitem): item.ihook.pytest_runtest_logreport(report=report) else: # failure detected and reruns not exhausted, since i < reruns + max_suite_reruns = item.session.config.option.max_suite_retries + if max_suite_reruns is not None: + suite_count = db.increment_suite_reruns() + if suite_count > max_suite_reruns: + # suite-wide limit exhausted — log as final failure + item.ihook.pytest_runtest_logreport(report=report) + continue + report.outcome = "rerun" time.sleep(delay) diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index b60716c..6f3f398 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1526,3 +1526,75 @@ def test_pass(): result = testdir.runpytest("--reruns-mode", "bogus") assert result.ret != 0 + + +def test_max_suite_retries_caps_total_reruns(testdir): + """Suite limit stops reruns once the total across all tests is reached.""" + testdir.makepyfile( + """ + def test_fail_1(): + assert False + + def test_fail_2(): + assert False + + def test_fail_3(): + assert False + """ + ) + # 3 tests each allowed up to 3 reruns, but suite cap is 4 total + result = testdir.runpytest("--reruns", "3", "--max-suite-retries", "4") + outcomes = result.parseoutcomes() + assert outcomes.get("rerun", 0) == 4 + assert outcomes.get("failed", 0) == 3 + + +def test_max_suite_retries_does_not_limit_when_sufficient(testdir): + """Suite limit has no effect when total reruns stay below the cap.""" + testdir.makepyfile( + """ + def test_fail(): + assert False + """ + ) + result = testdir.runpytest("--reruns", "2", "--max-suite-retries", "10") + assert_outcomes(result, passed=0, failed=1, rerun=2) + + +def test_max_suite_retries_zero_disables_all_reruns(testdir): + """Suite limit of 0 prevents any reruns from occurring.""" + testdir.makepyfile( + """ + def test_fail(): + assert False + """ + ) + result = testdir.runpytest("--reruns", "3", "--max-suite-retries", "0") + assert_outcomes(result, passed=0, failed=1, rerun=0) + + +def test_max_suite_retries_works_with_passing_tests(testdir): + """Suite limit only counts actual reruns, not passing test runs.""" + testdir.makepyfile( + """ + def test_pass(): + assert True + + def test_fail(): + assert False + """ + ) + result = testdir.runpytest("--reruns", "3", "--max-suite-retries", "2") + assert_outcomes(result, passed=1, failed=1, rerun=2) + + +def test_max_suite_retries_without_reruns_has_no_effect(testdir): + """--max-suite-retries alone (without --reruns) does not break anything.""" + testdir.makepyfile( + """ + def test_fail(): + assert False + """ + ) + result = testdir.runpytest("--max-suite-retries", "5") + assert_outcomes(result, passed=0, failed=1, rerun=0) From 8b3e3304ccd5f3a0ed745d515e42a286911bfb8c Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Thu, 4 Jun 2026 07:13:17 -0600 Subject: [PATCH 02/11] fix: use atomic try-increment-if-below-cap for suite rerun counter [resolve #3] Review by @Copilot (PR #332): "This increments the suite-wide counter even when the cap is already exhausted, which can permanently overshoot the configured limit..." Challenge: evidence=VALID suggestion=VALID resolution=as-suggested - Add `try_increment_suite_reruns(max_cap)` to StatusDB, ServerStatusDB, ClientStatusDB - ServerStatusDB adds `try_inc` socket protocol (atomic check-then-increment) - Caller uses bool return instead of post-increment comparison - Counter now reflects reruns actually performed, not attempts --- Co-authored-by: Claude Code Co-authored-by: OpenAI Codex --- src/pytest_rerunfailures.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 366f6bd..8dbf5bb 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -448,6 +448,13 @@ def increment_suite_reruns(self) -> int: self._suite_rerun_count += 1 return self._suite_rerun_count + def try_increment_suite_reruns(self, max_cap: int) -> bool: + with self._suite_lock: + if self._suite_rerun_count < max_cap: + self._suite_rerun_count += 1 + return True + return False + def get_suite_reruns(self) -> int: """Return the current suite-wide rerun count.""" return self._suite_rerun_count @@ -540,6 +547,14 @@ def run_connection(self, conn): new_v = self._get(i, k) + 1 self._set(i, k, new_v) self._sock_send(conn, str(new_v)) + elif op == "try_inc": + with self._suite_lock: + current = self._get(i, k) + if current < int(v): + self._set(i, k, current + 1) + self._sock_send(conn, "1") + else: + self._sock_send(conn, "0") def _set(self, i: str, k: str, v: int): if i not in self.rerunfailures_db: @@ -559,6 +574,14 @@ def increment_suite_reruns(self) -> int: self._set("__suite__", "r", new_v) return new_v + def try_increment_suite_reruns(self, max_cap: int) -> bool: + with self._suite_lock: + current = self._get("__suite__", "r") + if current < max_cap: + self._set("__suite__", "r", current + 1) + return True + return False + def get_suite_reruns(self) -> int: """Return the current suite-wide rerun count.""" return self._get("__suite__", "r") @@ -581,6 +604,12 @@ def increment_suite_reruns(self) -> int: self._sock_send(self.sock, "|".join(("inc", "__suite__", "r", ""))) return int(self._sock_recv(self.sock)) + def try_increment_suite_reruns(self, max_cap: int) -> bool: + self._sock_send( + self.sock, "|".join(("try_inc", "__suite__", "r", str(max_cap))) + ) + return self._sock_recv(self.sock) == "1" + def get_suite_reruns(self) -> int: """Return the current suite-wide rerun count.""" return self._get("__suite__", "r") @@ -686,8 +715,7 @@ def pytest_runtest_protocol(item, nextitem): # failure detected and reruns not exhausted, since i < reruns max_suite_reruns = item.session.config.option.max_suite_retries if max_suite_reruns is not None: - suite_count = db.increment_suite_reruns() - if suite_count > max_suite_reruns: + if not db.try_increment_suite_reruns(max_suite_reruns): # suite-wide limit exhausted — log as final failure item.ihook.pytest_runtest_logreport(report=report) continue From 0aeec5540195cbe24dfab9461d02a0196b079e07 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Thu, 4 Jun 2026 07:14:10 -0600 Subject: [PATCH 03/11] fix: use public group.addoption for --max-suite-retries [resolve #1] Review by @Copilot (PR #332): "group._addoption is a private pytest API and can break across pytest versions." Challenge: evidence=VALID suggestion=VALID resolution=as-suggested --- Co-authored-by: Claude Code Co-authored-by: OpenAI Codex --- src/pytest_rerunfailures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 8dbf5bb..d611b38 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -120,7 +120,7 @@ def pytest_addoption(parser): "'rerun test summary info' section, which is emitted automatically " "when this flag is set.", ) - group._addoption( + group.addoption( "--max-suite-retries", action="store", dest="max_suite_retries", From c48a1986f856a7c04c351ee64df0f9a6b263ad74 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Thu, 4 Jun 2026 07:22:48 -0600 Subject: [PATCH 04/11] lint: auto-fix violations after resolve cycle --- Co-authored-by: Claude Code --- pyproject.toml | 5 +++++ src/pytest_rerunfailures.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4d6aa3d..a5afddc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,3 +72,8 @@ lint.pydocstyle.convention = "google" [tool.check-manifest] ignore = [ ".pre-commit-config.yaml" ] + +[dependency-groups] +dev = [ + "mypy>=2.1.0", +] diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index d611b38..a074976 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -10,6 +10,7 @@ import traceback import warnings from contextlib import suppress +from typing import Any import pytest from _pytest.outcomes import fail @@ -615,7 +616,7 @@ def get_suite_reruns(self) -> int: return self._get("__suite__", "r") -suspended_finalizers = {} +suspended_finalizers: dict[Any, Any] = {} def pytest_runtest_teardown(item, nextitem): From 9817e4b91cc8718be90d114efbb225d38ea4db3d Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Tue, 9 Jun 2026 19:57:43 +0200 Subject: [PATCH 05/11] refine: validate --max-suite-retries >= 0 at config time [resolve #5] Review by @Copilot (PR #332): "--max-suite-retries currently accepts negative integers (because `type=int`)..." Challenge: evidence=VALID suggestion=VALID resolution=as-suggested --- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: OpenAI Codex --- src/pytest_rerunfailures.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index a074976..579eccd 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -144,6 +144,11 @@ def check_options(config): if config.option.reruns != 0: if config.option.usepdb: # a core option raise pytest.UsageError("--reruns incompatible with --pdb") + if ( + config.option.max_suite_retries is not None + and config.option.max_suite_retries < 0 + ): + raise pytest.UsageError("--max-suite-retries must be >= 0") def _get_marker(item): From 9e46bcc07758b474f437232a091f6a614e839b6e Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:03:48 +0200 Subject: [PATCH 06/11] refine: read _suite_rerun_count under lock in get_suite_reruns [resolve #6] Review by @Copilot (PR #332): "get_suite_reruns() reads _suite_rerun_count without the lock..." Challenge: evidence=VALID suggestion=VALID resolution=as-suggested --- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: OpenAI Codex --- src/pytest_rerunfailures.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 579eccd..e9d80d6 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -462,8 +462,9 @@ def try_increment_suite_reruns(self, max_cap: int) -> bool: return False def get_suite_reruns(self) -> int: - """Return the current suite-wide rerun count.""" - return self._suite_rerun_count + """Return the current suite-wide rerun count (read under lock for thread safety).""" + with self._suite_lock: + return self._suite_rerun_count def _hash(self, crashitem: str) -> str: if crashitem not in self.hmap: From e75460a7d1437479c23af87b5657efb194db5e21 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:09:46 +0200 Subject: [PATCH 07/11] test: use assert_outcomes() in test_max_suite_retries_caps_total_reruns [resolve #7] Review by @Copilot (PR #332): "This test manually inspects parseoutcomes() while the surrounding..." Challenge: evidence=VALID suggestion=REJECT resolution=self-resolved --- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: OpenAI Codex --- tests/test_pytest_rerunfailures.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 6f3f398..3b0ea45 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1544,9 +1544,7 @@ def test_fail_3(): ) # 3 tests each allowed up to 3 reruns, but suite cap is 4 total result = testdir.runpytest("--reruns", "3", "--max-suite-retries", "4") - outcomes = result.parseoutcomes() - assert outcomes.get("rerun", 0) == 4 - assert outcomes.get("failed", 0) == 3 + assert_outcomes(result, passed=0, failed=3, rerun=4) def test_max_suite_retries_does_not_limit_when_sufficient(testdir): From abdf2ea6a1d7dc4917c1370c90027d10212d3080 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:13:15 +0200 Subject: [PATCH 08/11] lint: type annotations + mypy config for suite-retries additions --- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> --- pyproject.toml | 17 +++++++++++++---- src/pytest_rerunfailures.py | 19 +++++++++++-------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a5afddc..c74c688 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,11 @@ urls = { Homepage = "https://github.com/pytest-dev/pytest-rerunfailures" } entry-points.pytest11.rerunfailures = "pytest_rerunfailures" +[dependency-groups] +dev = [ + "mypy>=2.1", +] + [tool.setuptools.dynamic] readme = { file = [ "HEADER.rst", "README.rst", "CHANGES.rst" ] } @@ -73,7 +78,11 @@ lint.pydocstyle.convention = "google" [tool.check-manifest] ignore = [ ".pre-commit-config.yaml" ] -[dependency-groups] -dev = [ - "mypy>=2.1.0", -] +[tool.mypy] +python_version = "3.10" +warn_return_any = true +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = [ "xdist.newhooks" ] +ignore_missing_imports = true diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index e9d80d6..c28899a 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -442,11 +442,11 @@ def pytest_handlecrashitem(self, crashitem, report, sched): # and failures (set after each failure or crash) # accessible from both the master and worker class StatusDB: - def __init__(self): - self.delim = b"\n" - self.hmap = {} - self._suite_rerun_count = 0 - self._suite_lock = threading.Lock() + def __init__(self) -> None: + self.delim: bytes = b"\n" + self.hmap: dict[str, str] = {} + self._suite_rerun_count: int = 0 + self._suite_lock: threading.Lock = threading.Lock() def increment_suite_reruns(self) -> int: """Atomically increment the suite-wide rerun counter; return new total.""" @@ -462,7 +462,10 @@ def try_increment_suite_reruns(self, max_cap: int) -> bool: return False def get_suite_reruns(self) -> int: - """Return the current suite-wide rerun count (read under lock for thread safety).""" + """Return the current suite-wide rerun count. + + Reads under lock for thread safety. + """ with self._suite_lock: return self._suite_rerun_count @@ -521,12 +524,12 @@ def _sock_send(self, conn, msg: str): class ServerStatusDB(SocketDB): - def __init__(self): + def __init__(self) -> None: super().__init__() self.sock.bind(("127.0.0.1", 0)) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.rerunfailures_db = {} + self.rerunfailures_db: dict[str, dict[str, int]] = {} t = threading.Thread(target=self.run_server, daemon=True) t.start() From cd2d6f2db8beee86feacf4a75a0f759cd668f6d9 Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:58:05 +0200 Subject: [PATCH 09/11] fix(suite-reruns): enforce suite rerun cap and option semantics - enforce suite-wide rerun cap semantics for xdist crash retries and fixture teardown paths - validate max-suite-reruns option consistently (including collect-only mode) - align CLI/docs/tests with renamed option and marker-driven rerun cap coverage --- Co-authored-by: Codex --- CHANGES.rst | 2 +- README.rst | 7 +- src/pytest_rerunfailures.py | 86 ++++++++++++++++----- tests/test_pytest_rerunfailures.py | 116 ++++++++++++++++++++++++++--- 4 files changed, 177 insertions(+), 34 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4128c23..3b392a5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,7 +7,7 @@ Changelog Features ++++++++ -- Add ``--max-suite-retries`` option to cap the total number of reruns across +- Add ``--max-suite-reruns`` option to cap the total number of reruns across the entire test suite. Once the limit is reached, no further reruns occur regardless of per-test ``--reruns`` or ``@pytest.mark.flaky`` settings. Fixes `#298 `_. diff --git a/README.rst b/README.rst index d8d9a6a..a5f8f44 100644 --- a/README.rst +++ b/README.rst @@ -224,16 +224,17 @@ Limit total reruns across the suite ------------------------------------ To cap the total number of reruns across the entire test suite regardless of -how many individual tests fail, pass ``--max-suite-retries``. Once the limit +how many individual tests fail, pass ``--max-suite-reruns``. Once the limit is reached, no further reruns occur even if individual tests have remaining retries: .. code-block:: bash - $ pytest --reruns 3 --max-suite-retries 10 + $ pytest --reruns 3 --max-suite-reruns 10 This is useful in large test suites to bound resource usage when many tests -are flaky at the same time. +are flaky at the same time. The cap applies after rerun selection, including +tests configured with ``--force-reruns`` and ``@pytest.mark.flaky``. Show tracebacks for retried failures ------------------------------------ diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index c28899a..82b04fe 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -122,9 +122,9 @@ def pytest_addoption(parser): "when this flag is set.", ) group.addoption( - "--max-suite-retries", + "--max-suite-reruns", action="store", - dest="max_suite_retries", + dest="max_suite_reruns", type=int, default=None, help="Maximum total number of reruns across the entire test suite. " @@ -140,15 +140,14 @@ def pytest_addoption(parser): # should run before / at the beginning of pytest_cmdline_main def check_options(config): val = config.getvalue - if not val("collectonly"): - if config.option.reruns != 0: - if config.option.usepdb: # a core option - raise pytest.UsageError("--reruns incompatible with --pdb") - if ( - config.option.max_suite_retries is not None - and config.option.max_suite_retries < 0 - ): - raise pytest.UsageError("--max-suite-retries must be >= 0") + if ( + config.option.max_suite_reruns is not None + and config.option.max_suite_reruns < 0 + ): + raise pytest.UsageError("--max-suite-reruns must be >= 0") + if not val("collectonly") and config.option.reruns != 0: + if config.option.usepdb: # a core option + raise pytest.UsageError("--reruns incompatible with --pdb") def _get_marker(item): @@ -394,6 +393,7 @@ def pytest_configure(config): "to 'reruns' times. Add a delay of 'reruns_delay' seconds " "between re-runs.", ) + check_options(config) if config.pluginmanager.hasplugin("xdist") and HAS_PYTEST_HANDLECRASHITEM: config.pluginmanager.register(XDistHooks()) @@ -414,11 +414,23 @@ def pytest_handlecrashitem(self, crashitem, report, sched): """Return the crashitem from pending and collection.""" db = sched.config.failures_db reruns = db.get_test_reruns(crashitem) + reserved_suite_rerun = False if db.get_test_failures(crashitem) < reruns: + max_suite_reruns = sched.config.option.max_suite_reruns + if max_suite_reruns is None: + cap_available = True + else: + reserved_suite_rerun = db.try_increment_suite_reruns(max_suite_reruns) + cap_available = reserved_suite_rerun + else: + cap_available = False + if cap_available: try: sched.mark_test_pending(crashitem) report.outcome = "rerun" except NotImplementedError: + if reserved_suite_rerun: + db.decrement_suite_reruns() # Some schedulers (like LoadScopeScheduling) don't implement # mark_test_pending # In this case, we can't reschedule the crashed test for rerun @@ -455,12 +467,19 @@ def increment_suite_reruns(self) -> int: return self._suite_rerun_count def try_increment_suite_reruns(self, max_cap: int) -> bool: + """Increment the suite counter when it is below the configured cap.""" with self._suite_lock: if self._suite_rerun_count < max_cap: self._suite_rerun_count += 1 return True return False + def decrement_suite_reruns(self) -> None: + """Release a suite slot when a scheduled rerun cannot be started.""" + with self._suite_lock: + if self._suite_rerun_count > 0: + self._suite_rerun_count -= 1 + def get_suite_reruns(self) -> int: """Return the current suite-wide rerun count. @@ -565,6 +584,12 @@ def run_connection(self, conn): self._sock_send(conn, "1") else: self._sock_send(conn, "0") + elif op == "dec": + with self._suite_lock: + current = self._get(i, k) + if current > 0: + self._set(i, k, current - 1) + self._sock_send(conn, "1") def _set(self, i: str, k: str, v: int): if i not in self.rerunfailures_db: @@ -585,6 +610,7 @@ def increment_suite_reruns(self) -> int: return new_v def try_increment_suite_reruns(self, max_cap: int) -> bool: + """Increment the suite counter when it is below the configured cap.""" with self._suite_lock: current = self._get("__suite__", "r") if current < max_cap: @@ -592,6 +618,13 @@ def try_increment_suite_reruns(self, max_cap: int) -> bool: return True return False + def decrement_suite_reruns(self) -> None: + """Release a suite slot when a scheduled rerun cannot be started.""" + with self._suite_lock: + current = self._get("__suite__", "r") + if current > 0: + self._set("__suite__", "r", current - 1) + def get_suite_reruns(self) -> int: """Return the current suite-wide rerun count.""" return self._get("__suite__", "r") @@ -615,11 +648,17 @@ def increment_suite_reruns(self) -> int: return int(self._sock_recv(self.sock)) def try_increment_suite_reruns(self, max_cap: int) -> bool: + """Increment the suite counter when it is below the configured cap.""" self._sock_send( self.sock, "|".join(("try_inc", "__suite__", "r", str(max_cap))) ) return self._sock_recv(self.sock) == "1" + def decrement_suite_reruns(self) -> None: + """Release a suite slot when a scheduled rerun cannot be started.""" + self._sock_send(self.sock, "|".join(("dec", "__suite__", "r", ""))) + self._sock_recv(self.sock) + def get_suite_reruns(self) -> int: """Return the current suite-wide rerun count.""" return self._get("__suite__", "r") @@ -628,6 +667,12 @@ def get_suite_reruns(self) -> int: suspended_finalizers: dict[Any, Any] = {} +def _restore_suspended_finalizers(item): + """Restore higher-scope finalizers after a rerun is not scheduled.""" + item.session._setupstate.stack.update(suspended_finalizers) + suspended_finalizers.clear() + + def pytest_runtest_teardown(item, nextitem): reruns = get_reruns_count(item) if reruns is None: @@ -642,6 +687,14 @@ def pytest_runtest_teardown(item, nextitem): _test_failed_statuses = getattr(item, "_test_failed_statuses", {}) + max_suite_reruns = item.session.config.option.max_suite_reruns + if ( + max_suite_reruns is not None + and item.session.config.failures_db.get_suite_reruns() >= max_suite_reruns + ): + _restore_suspended_finalizers(item) + return + # Only remove non-function level actions from the stack if the test is to be re-run # Exceeding re-run limits, being free of failue statuses, and encountering # allowable exceptions indicate that the test is not to be re-ran. @@ -662,8 +715,7 @@ def pytest_runtest_teardown(item, nextitem): del item.session._setupstate.stack[key] else: # restore suspended finalizers - item.session._setupstate.stack.update(suspended_finalizers) - suspended_finalizers.clear() + _restore_suspended_finalizers(item) @pytest.hookimpl(hookwrapper=True) @@ -698,9 +750,6 @@ def pytest_runtest_protocol(item, nextitem): # flaky return - # while this doesn't need to be run with every item, it will fail on the - # first item if necessary - check_options(item.session.config) delay = get_reruns_delay(item) parallel = not is_master(item.config) db = item.session.config.failures_db @@ -723,10 +772,11 @@ def pytest_runtest_protocol(item, nextitem): item.ihook.pytest_runtest_logreport(report=report) else: # failure detected and reruns not exhausted, since i < reruns - max_suite_reruns = item.session.config.option.max_suite_retries + max_suite_reruns = item.session.config.option.max_suite_reruns if max_suite_reruns is not None: if not db.try_increment_suite_reruns(max_suite_reruns): - # suite-wide limit exhausted — log as final failure + # Suite-wide limit exhausted -- log as final failure. + _restore_suspended_finalizers(item) item.ihook.pytest_runtest_logreport(report=report) continue diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 3b0ea45..1146fa4 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1,10 +1,11 @@ import random import time +from types import SimpleNamespace from unittest import mock import pytest -from pytest_rerunfailures import HAS_PYTEST_HANDLECRASHITEM +from pytest_rerunfailures import HAS_PYTEST_HANDLECRASHITEM, StatusDB, XDistHooks pytest_plugins = "pytester" @@ -273,6 +274,54 @@ def test_pass(): assert_outcomes(result, passed=2, rerun=1) +@pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") +def test_max_suite_reruns_caps_temporary_test_crash(testdir): + testdir.makepyfile( + f""" + def test_crash(): + {temporary_crash(2)} + + def test_pass(): + pass + """ + ) + result = testdir.runpytest( + "-p", + "xdist", + "-n", + "1", + "--reruns", + "3", + "--max-suite-reruns", + "1", + ) + assert result.ret != pytest.ExitCode.OK + check_outcome_field(result.parseoutcomes(), "rerun", 1) + + +def test_xdist_crash_rerun_releases_cap_when_scheduler_rejects(): + db = StatusDB() + db.get_test_reruns = lambda _: 1 + db.get_test_failures = lambda _: 0 + + def mark_test_pending(_): + raise NotImplementedError + + sched = SimpleNamespace( + config=SimpleNamespace( + failures_db=db, + option=SimpleNamespace(max_suite_reruns=1), + ), + mark_test_pending=mark_test_pending, + ) + report = SimpleNamespace(outcome="failed", longrepr=None) + + XDistHooks().pytest_handlecrashitem("test_crash", report, sched) + + assert report.outcome == "failed" + assert db.get_suite_reruns() == 0 + + def test_rerun_passes_after_temporary_test_failure_with_flaky_mark(testdir): testdir.makepyfile( f""" @@ -1528,7 +1577,7 @@ def test_pass(): assert result.ret != 0 -def test_max_suite_retries_caps_total_reruns(testdir): +def test_max_suite_reruns_caps_total_reruns(testdir): """Suite limit stops reruns once the total across all tests is reached.""" testdir.makepyfile( """ @@ -1543,11 +1592,11 @@ def test_fail_3(): """ ) # 3 tests each allowed up to 3 reruns, but suite cap is 4 total - result = testdir.runpytest("--reruns", "3", "--max-suite-retries", "4") + result = testdir.runpytest("--reruns", "3", "--max-suite-reruns", "4") assert_outcomes(result, passed=0, failed=3, rerun=4) -def test_max_suite_retries_does_not_limit_when_sufficient(testdir): +def test_max_suite_reruns_does_not_limit_when_sufficient(testdir): """Suite limit has no effect when total reruns stay below the cap.""" testdir.makepyfile( """ @@ -1555,11 +1604,11 @@ def test_fail(): assert False """ ) - result = testdir.runpytest("--reruns", "2", "--max-suite-retries", "10") + result = testdir.runpytest("--reruns", "2", "--max-suite-reruns", "10") assert_outcomes(result, passed=0, failed=1, rerun=2) -def test_max_suite_retries_zero_disables_all_reruns(testdir): +def test_max_suite_reruns_zero_disables_all_reruns(testdir): """Suite limit of 0 prevents any reruns from occurring.""" testdir.makepyfile( """ @@ -1567,11 +1616,11 @@ def test_fail(): assert False """ ) - result = testdir.runpytest("--reruns", "3", "--max-suite-retries", "0") + result = testdir.runpytest("--reruns", "3", "--max-suite-reruns", "0") assert_outcomes(result, passed=0, failed=1, rerun=0) -def test_max_suite_retries_works_with_passing_tests(testdir): +def test_max_suite_reruns_works_with_passing_tests(testdir): """Suite limit only counts actual reruns, not passing test runs.""" testdir.makepyfile( """ @@ -1582,17 +1631,60 @@ def test_fail(): assert False """ ) - result = testdir.runpytest("--reruns", "3", "--max-suite-retries", "2") + result = testdir.runpytest("--reruns", "3", "--max-suite-reruns", "2") assert_outcomes(result, passed=1, failed=1, rerun=2) -def test_max_suite_retries_without_reruns_has_no_effect(testdir): - """--max-suite-retries alone (without --reruns) does not break anything.""" +def test_max_suite_reruns_without_reruns_has_no_effect(testdir): + """--max-suite-reruns alone (without --reruns) does not break anything.""" testdir.makepyfile( """ def test_fail(): assert False """ ) - result = testdir.runpytest("--max-suite-retries", "5") + result = testdir.runpytest("--max-suite-reruns", "5") assert_outcomes(result, passed=0, failed=1, rerun=0) + + +def test_max_suite_reruns_caps_flaky_marker_reruns(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.mark.flaky(reruns=3) + def test_fail(): + assert False + """ + ) + result = testdir.runpytest("--max-suite-reruns", "2") + assert_outcomes(result, passed=0, failed=1, rerun=2) + + +def test_max_suite_reruns_rejects_negative_without_reruns(testdir): + testdir.makepyfile("def test_pass(): pass") + result = testdir.runpytest("--max-suite-reruns", "-1") + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines("*--max-suite-reruns must be >= 0*") + + collect_result = testdir.runpytest("--collect-only", "--max-suite-reruns", "-1") + assert collect_result.ret == pytest.ExitCode.USAGE_ERROR + collect_result.stderr.fnmatch_lines("*--max-suite-reruns must be >= 0*") + + +def test_max_suite_reruns_preserves_fixture_teardown_when_exhausted(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", autouse=True) + def module_fixture(): + yield + print("module teardown") + + def test_fail(): + assert False + """ + ) + result = testdir.runpytest("-s", "--reruns", "1", "--max-suite-reruns", "0") + result.stdout.fnmatch_lines("*module teardown*") From e80680a0fcedfb34033ef7d51ebe25b5f75681ce Mon Sep 17 00:00:00 2001 From: jirka <6035284+Borda@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:04:45 +0200 Subject: [PATCH 10/11] fix(suite-reruns): harden suite-rerun typing and coverage - Guard subtest fallback imports with mypy-safe aliases for pytest < 9 compatibility. - Add regression coverage for --max-suite-reruns capping --force-reruns behavior. --- Co-authored-by: Codex --- src/pytest_rerunfailures.py | 13 ++++++++++--- tests/test_pytest_rerunfailures.py | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index caf7d67..7ff8026 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -17,13 +17,20 @@ from _pytest.runner import runtestprotocol from packaging.version import parse as parse_version +failed_subtests_key: Any +SubtestReport: Any +_failed_subtests_key: Any = None +_SubtestReport: Any = None + try: - from _pytest.subtests import SubtestReport, failed_subtests_key + from _pytest.subtests import SubtestReport as _SubtestReport + from _pytest.subtests import failed_subtests_key as _failed_subtests_key except ImportError: if pytest.version_tuple >= (9, 0, 0): raise - failed_subtests_key = None - SubtestReport = None + +failed_subtests_key = _failed_subtests_key +SubtestReport = _SubtestReport try: from xdist.newhooks import pytest_handlecrashitem diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index ea7651c..207a1cb 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1694,6 +1694,23 @@ def test_fail_3(): assert_outcomes(result, passed=0, failed=3, rerun=4) +def test_max_suite_reruns_caps_force_reruns(testdir): + """Suite cap applies after ``--force-reruns`` selection.""" + testdir.makepyfile( + """ + def test_fail_1(): + assert False + + def test_fail_2(): + assert False + """ + ) + # Force reruns allows every failing test to rerun, but the suite cap should + # limit total reruns to one. + result = testdir.runpytest("--force-reruns", "5", "--max-suite-reruns", "1") + assert_outcomes(result, passed=0, failed=2, rerun=1) + + def test_max_suite_reruns_does_not_limit_when_sufficient(testdir): """Suite limit has no effect when total reruns stay below the cap.""" testdir.makepyfile( @@ -1765,7 +1782,7 @@ def test_subtests(subtests): @pytest.mark.skipif(not has_subtests, reason="Only supported on pytest 9.0 and newer") def test_too_many_failing_subtests_are_failures(testdir): testdir.makepyfile( - f""" + """ import pytest def test_subtests(subtests): From 82e5f9c554a396b010692e4c412b3280ebd1d64d Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:35:23 +0200 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.rst | 2 +- src/pytest_rerunfailures.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8ddfd23..76194c7 100644 --- a/README.rst +++ b/README.rst @@ -244,7 +244,7 @@ Limit total reruns across the suite To cap the total number of reruns across the entire test suite regardless of how many individual tests fail, pass ``--max-suite-reruns``. Once the limit is reached, no further reruns occur even if individual tests have remaining -retries: +reruns: .. code-block:: bash diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 7ff8026..490a19f 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -614,7 +614,7 @@ def __init__(self) -> None: self.delim: bytes = b"\n" self.hmap: dict[str, str] = {} self._suite_rerun_count: int = 0 - self._suite_lock: threading.Lock = threading.Lock() + self._suite_lock = threading.Lock() def increment_suite_reruns(self) -> int: """Atomically increment the suite-wide rerun counter; return new total."""