From d27b173a55fadce7afa3b056b20437f40f424e86 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 19 Aug 2026 18:10:54 -0400 Subject: [PATCH] test(runner): make the lease-expiry test observe the clock, not race it `test_expired_unrenewed_lease_cannot_report_false_success` gave the lease an 80 ms life and slept 120 ms inside the run. The runner refuses a dispatch whose lease is already dead at the pre-start check, and bundle staging plus policy binding sit between the lease read and that check. When those took longer than 80 ms the job was refused before execution, `flow.calls` stayed empty, and the test failed with `assert 0 == 1`. It did exactly that on `main` for ubuntu-latest / Python 3.11. Drive the sequence by observation instead: - Patch `engine.runner_loop.datetime` with a real clock plus a test-controlled offset, and give the lease 30 s so the pre-start check always passes. - The patched run body waits for the renew loop's first extend attempt, then moves the clock past the deadline, then holds the run open until the renew loop stops attempting extends, which proves it observed the expiry. Every assertion is unchanged: the run executed once, at least one extend was attempted, the ack is `uncertain`, no `run_summary` reached the wire, and the journal records `uncertain`. The test now takes about 0.16 s instead of 0.12 s of fixed sleeps, and passed five consecutive local runs. Tests: full suite 951 passed, 6 skipped. Ruff passed. Co-Authored-By: Claude Opus 5 --- tests/test_engine/test_runner_loop.py | 39 +++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_engine/test_runner_loop.py b/tests/test_engine/test_runner_loop.py index acb8705..0b31126 100644 --- a/tests/test_engine/test_runner_loop.py +++ b/tests/test_engine/test_runner_loop.py @@ -1114,20 +1114,55 @@ async def test_expired_unrenewed_lease_cannot_report_false_success( login() _bundle, digest = make_bundle(config) cloud.extend_status = 503 + + class OffsetClock(datetime): + """The real clock plus a test-controlled offset. + + The lease must be alive at the pre-start check and dead during the + run. Expressing that as two short wall-clock sleeps races with slow + bundle staging or policy binding, so the run itself moves the clock + instead. + """ + + offset = timedelta() + + @classmethod + def now(cls, tz=None): # type: ignore[override] + return datetime.now(tz) + cls.offset + job = make_job(digest) job["lease"] = { "job_id": "job_1", "visibility_timeout_s": 900, - "expires_at": (datetime.now(UTC) + timedelta(seconds=0.08)).isoformat(), + # Comfortably alive at start; only the offset below expires it. + "expires_at": (datetime.now(UTC) + timedelta(seconds=30)).isoformat(), } original_run = flow.run def slow_run(*args, **kwargs): - time.sleep(0.12) + # This body runs on a worker thread while the renew loop owns the + # event loop, so it can drive the sequence by observation instead + # of by sleeping for a guessed duration. + guard = time.monotonic() + 5.0 + # 1. Let the renew loop attempt at least one extend against the + # still-live lease. The fake cloud answers 503 every time. + while not cloud.extends and time.monotonic() < guard: + time.sleep(0.005) + # 2. Push every later engine.runner_loop clock read past the lease + # deadline. + OffsetClock.offset = timedelta(seconds=60) + # 3. Hold the run open until the renew loop has seen the expiry and + # returned, which it proves by attempting no further extend. + while time.monotonic() < guard: + attempts = len(cloud.extends) + time.sleep(0.05) + if len(cloud.extends) == attempts: + break return original_run(*args, **kwargs) monkeypatch.setattr(flow, "run", slow_run) monkeypatch.setattr("engine.runner_loop.LEASE_EXTEND_INTERVAL_S", 0.01) + monkeypatch.setattr("engine.runner_loop.datetime", OffsetClock) async with svc._http_factory() as http: client = RunnerClient(http, token="oar_test")