diff --git a/README.md b/README.md index 3f1531f..bf4e1be 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,106 @@ # simloop -Deterministic simulation testing for Python asyncio — seeded task scheduling, +Deterministic simulation testing for Python asyncio: seeded scheduling, virtual time, and a simulated network with fault injection. Any failure -it finds replays exactly from a seed. +simloop finds replays exactly from a seed. Rust has [madsim](https://github.com/madsim-rs/madsim) and -[turmoil](https://github.com/tokio-rs/turmoil). Python has nothing comparable. -simloop is that missing tool: it runs **real, unmodified asyncio code** on a -simulated event loop where every scheduling decision is drawn from a seeded PRNG -and time is virtual — so a test that passes 999 times and deadlocks once becomes -a test that deadlocks *every* time at seed 4217, in milliseconds, under a debugger. +[turmoil](https://github.com/tokio-rs/turmoil); FoundationDB and +TigerBeetle made the technique famous. simloop brings it to asyncio: +your real, unmodified networking code runs on a simulated loop where +time is virtual, every scheduling decision is seeded, and the network +loses, delays, duplicates, and partitions traffic on command. -## Status +## Install -**Early development.** The deterministic loop core works: seeded scheduling, -virtual time, replay-proving trace hashes, deadlock detection, and seeded -`sim.random` / `sim.uuid4` / `sim.time` shims, with the stdlib coordination -primitives (`Queue`, `Event`, `Lock`, `gather`, `TaskGroup`, `timeout`) -running unchanged on top. The simulated network is in: `asyncio.open_connection`, -`start_server`, and datagram endpoints run unchanged over in-memory transports -with seeded latency, drops, duplication, partitions, and host crashes. See -[the supported subset](docs/supported-api.md) for the exact contract. +``` +pip install simloop +``` -## Planned +Python 3.12+. No runtime dependencies. The pytest plugin ships in the +same package and activates automatically. -- **SimLoop** — a deterministic `asyncio` event loop: virtual clock, seeded - ready-queue ordering, append-only scheduling trace with replay-proving hashes. -- **pytest plugin** — `@sim_test(seeds=1000)` to explore schedules, and an exact - replay flag for any failing seed. +## Find a bug, then replay it -Design notes live in [`docs/`](docs/). +```python +import asyncio +from simloop import sim_test + + +@sim_test(seeds=200) +async def test_replies_survive_a_lossy_network(): + loop = asyncio.get_running_loop() + loop.net.set_defaults(latency=(0.001, 0.050)) + + async def serve(): + async def handle(reader, writer): + writer.write((await reader.readline()).upper()) + writer.close() + + server = await asyncio.start_server(handle, port=8080) + async with server: + await server.serve_forever() + + loop.net.host("server").create_task(serve()) + await asyncio.sleep(1.0) + + async with asyncio.timeout(30.0): + reader, writer = await asyncio.open_connection("server", 8080) + writer.write(b"hello\n") + assert await reader.readline() == b"HELLO\n" +``` + +`@sim_test(seeds=200)` runs the test under 200 seeds, each on a fresh +simulated loop, and stops at the first failure: + +``` +simloop: failed at seed 41 (41 seeds passed first) +replay: pytest 'tests/test_echo.py::test_replies_survive_a_lossy_network' --simloop-replay=41 + +last 20 trace events: + [t=1.0312] net seq=812 send driver>server + ... +pending tasks by host: + server Task 'Task-2' awaiting serve_forever at ... +``` + +The replay command reproduces the failure exactly — same scheduling +decisions, same fault decisions, same trace. In CI, crank the search +without touching code: + +``` +pytest --simloop-seeds=1000 +``` + +## What the simulation gives you + +- **Seeded scheduling** — the ready queue's execution order comes from a + per-run PRNG; a seed pins the entire interleaving. +- **Virtual time** — `asyncio.sleep(300)` costs nothing; timeouts fire + in simulated seconds. +- **A simulated network** — `open_connection` / `start_server` / + datagram endpoints run over an in-memory packet core with per-link + latency, drop, and duplication, plus partitions and host crashes: + +```python +net = loop.net +net.set_defaults(latency=(0.001, 0.010), drop=0.05) +net.partition({"node1"}, {"node2", "node3"}) # silent blackhole +loop.call_later(5.0, net.heal) # heals in virtual time +net.crash("node2") # no reset, just silence +``` + +- **Replayable traces** — every scheduling and fault decision lands in + an append-only trace whose hash proves a replay is exact. + +## Honest limits + +Code that goes through the event-loop API is supported; code that +bypasses it is fenced: threads and executors, raw sockets, subprocesses, +signals, TLS, and `getaddrinfo` raise `SimulationFenceError` rather than +silently breaking determinism. Write-side flow control is not simulated. +The full contract is in [docs/supported-api.md](docs/supported-api.md). ## License -[MIT](LICENSE) +MIT diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..c6481d5 --- /dev/null +++ b/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ["pytester"] diff --git a/pyproject.toml b/pyproject.toml index 294ef17..6ebc287 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "simloop" -version = "0.0.1.dev0" +version = "0.1.0" description = "Deterministic simulation testing for Python asyncio: seeded scheduling, virtual time, replayable failures" readme = "README.md" requires-python = ">=3.12" @@ -9,7 +9,7 @@ license-files = ["LICENSE"] authors = [{ name = "Dhruv Kumar Singh", email = "dsingh32@binghamton.edu" }] keywords = ["asyncio", "testing", "deterministic", "simulation", "dst"] classifiers = [ - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 3 - Alpha", "Framework :: AsyncIO", "Intended Audience :: Developers", "Programming Language :: Python :: 3.12", @@ -22,6 +22,9 @@ dependencies = [] [dependency-groups] dev = ["pytest>=8", "mypy>=1.11"] +[project.entry-points.pytest11] +simloop = "simloop._pytest_plugin" + [build-system] requires = ["hatchling>=1.27"] build-backend = "hatchling.build" diff --git a/src/simloop/__init__.py b/src/simloop/__init__.py index 53e2e45..cb001ba 100644 --- a/src/simloop/__init__.py +++ b/src/simloop/__init__.py @@ -1,14 +1,16 @@ """simloop — deterministic simulation testing for Python asyncio.""" +from simloop._explore import SeedReport, explore, sim_test from simloop._loop import SimLoop, SimulationDeadlockError, SimulationFenceError from simloop._net import Host, SimNetwork from simloop._sim import Sim, sim from simloop._trace import TraceEvent -__version__ = "0.0.1.dev0" +__version__ = "0.1.0" __all__ = [ "Host", + "SeedReport", "Sim", "SimLoop", "SimNetwork", @@ -16,5 +18,7 @@ "SimulationFenceError", "TraceEvent", "__version__", + "explore", "sim", + "sim_test", ] diff --git a/src/simloop/_explore.py b/src/simloop/_explore.py new file mode 100644 index 0000000..2ec4d06 --- /dev/null +++ b/src/simloop/_explore.py @@ -0,0 +1,234 @@ +"""Run a simulation test under many seeds and report the first failure. + +This module is plain library code: it never imports pytest. The pytest +integration in ``_pytest_plugin`` feeds session options in through the +module-level ``overrides`` object, which keeps ``import simloop`` free of +any test-framework dependency. +""" + +from __future__ import annotations + +import asyncio +import functools +import os +from collections.abc import Callable, Coroutine, Iterable +from dataclasses import dataclass +from typing import Any, overload + +from simloop._loop import SimLoop +from simloop._trace import TraceEvent + + +@dataclass(frozen=True, slots=True) +class PendingTask: + """One task still pending when a seed failed.""" + + host: str + name: str + awaiting: str + where: str + + +@dataclass(frozen=True, slots=True) +class SeedReport: + """Everything known about the first failing seed.""" + + seed: int + seeds_passed: int + exception: Exception + trace_events: tuple[TraceEvent, ...] + trace_hash: str + pending: tuple[PendingTask, ...] + + def render(self, test_id: str | None = None) -> str: + lines = [ + f"simloop: failed at seed {self.seed} " + f"({self.seeds_passed} seeds passed first)" + ] + if test_id is not None: + lines.append( + f"replay: pytest '{test_id}' --simloop-replay={self.seed}" + ) + if self.trace_events: + lines.append("") + lines.append(f"last {len(self.trace_events)} trace events:") + for event in self.trace_events: + lines.append( + f" [t={event.when:.4f}] {event.kind:<8} " + f"seq={event.seq} {event.label}" + ) + if self.pending: + lines.append("pending tasks by host:") + for task in self.pending: + lines.append( + f" {task.host} Task {task.name!r} " + f"awaiting {task.awaiting} at {task.where}" + ) + return "\n".join(lines) + + +def explore( + fn: Callable[[], Coroutine[Any, Any, object]], + seeds: Iterable[int], + *, + trace_tail: int = 20, +) -> SeedReport | None: + """Run ``fn`` once per seed on a fresh SimLoop; stop at the first failure. + + Returns a :class:`SeedReport` for the first seed whose run raised an + ``Exception``, or ``None`` when every seed passed. ``BaseException``s + that are not test failures (``KeyboardInterrupt``, ``SystemExit``) + propagate immediately. + """ + passed = 0 + for seed in seeds: + loop = SimLoop(seed) + try: + try: + loop.run_until_complete(fn()) + except Exception as exc: + return SeedReport( + seed=seed, + seeds_passed=passed, + exception=exc, + trace_events=loop.trace[-trace_tail:] if trace_tail else (), + trace_hash=loop.trace_hash(), + pending=_pending_tasks(loop), + ) + finally: + _drain(loop) + finally: + loop.close() + passed += 1 + return None + + +def _pending_tasks(loop: SimLoop) -> tuple[PendingTask, ...]: + found: list[PendingTask] = [] + for host, tasks in loop.net._tasks.items(): + for task in tasks: + if task.done(): + continue + awaiting = "?" + where = "?" + stack = task.get_stack() + if stack: + frame = stack[-1] + awaiting = frame.f_code.co_name + where = f"{_short_path(frame.f_code.co_filename)}:{frame.f_lineno}" + found.append( + PendingTask( + host=host, name=task.get_name(), awaiting=awaiting, where=where + ) + ) + return tuple(found) + + +def _short_path(filename: str) -> str: + cwd = os.getcwd() + if filename.startswith(cwd + os.sep): + return filename[len(cwd) + 1 :] + return filename + + +def _drain(loop: SimLoop) -> None: + """Cancel tasks a finished run left pending and let them unwind. + + Without this, an abandoned task's garbage collection would route + "Task was destroyed but it is pending!" through the loop's exception + handler onto stderr long after the run ended. + """ + pending = [ + task + for tasks in loop.net._tasks.values() + for task in tasks + if not task.done() + ] + if not pending: + return + for task in pending: + task.cancel() + try: + loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + except Exception: + # The run is already over; teardown failures add nothing. + pass + + +@dataclass +class _Overrides: + """Session state the pytest plugin writes; consulted by sim_test wrappers. + + ``seeds`` and ``replay`` mirror the --simloop-* options; ``node_id`` is + the test currently running, so reports can print an exact replay + command. The counters feed the plugin's terminal summary. + """ + + seeds: int | None = None + replay: int | None = None + node_id: str | None = None + sim_tests: int = 0 + seeds_explored: int = 0 + + +overrides = _Overrides() + +_TestFn = Callable[..., Coroutine[Any, Any, object]] + + +@overload +def sim_test(fn: _TestFn, /) -> Callable[..., None]: ... + + +@overload +def sim_test( + *, seeds: int = ..., trace_tail: int = ... +) -> Callable[[_TestFn], Callable[..., None]]: ... + + +def sim_test( + fn: _TestFn | None = None, + /, + *, + seeds: int = 10, + trace_tail: int = 20, +) -> Callable[..., None] | Callable[[_TestFn], Callable[..., None]]: + """Turn an ``async def`` test into a seed-exploring synchronous test. + + The wrapper runs the coroutine under ``seeds`` seeds (0..N-1) via + :func:`explore` and re-raises the first failure with the rendered + report attached as an exception note. Under pytest, the --simloop-seeds + and --simloop-replay options override the decorator's arguments. + """ + if seeds < 1: + raise ValueError("seeds must be at least 1") + + def decorate(test_fn: _TestFn) -> Callable[..., None]: + @functools.wraps(test_fn) + def wrapper(*args: Any, **kwargs: Any) -> None: + if overrides.replay is not None: + seed_set: range | tuple[int, ...] = (overrides.replay,) + elif overrides.seeds is not None: + seed_set = range(overrides.seeds) + else: + seed_set = range(seeds) + if len(seed_set) < 1: + raise ValueError("seeds must be at least 1") + report = explore( + functools.partial(test_fn, *args, **kwargs), + seed_set, + trace_tail=trace_tail, + ) + overrides.sim_tests += 1 + if report is None: + overrides.seeds_explored += len(seed_set) + return + overrides.seeds_explored += report.seeds_passed + 1 + report.exception.add_note(report.render(overrides.node_id)) + raise report.exception + + return wrapper + + if fn is not None: + return decorate(fn) + return decorate diff --git a/src/simloop/_pytest_plugin.py b/src/simloop/_pytest_plugin.py new file mode 100644 index 0000000..e01ae85 --- /dev/null +++ b/src/simloop/_pytest_plugin.py @@ -0,0 +1,62 @@ +"""pytest integration: seed-count and replay options, replay lines. + +pytest loads this module through the ``pytest11`` entry point declared in +pyproject.toml. simloop itself never imports it, so the library keeps its +zero-dependency import surface. +""" + +from __future__ import annotations + +import pytest + +from simloop import _explore + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup("simloop") + group.addoption( + "--simloop-seeds", + type=int, + default=None, + metavar="N", + help="run every @sim_test under seeds 0..N-1, overriding decorators", + ) + group.addoption( + "--simloop-replay", + type=int, + default=None, + metavar="SEED", + help="run every @sim_test at exactly this seed", + ) + + +def pytest_configure(config: pytest.Config) -> None: + _explore.overrides.seeds = config.getoption("--simloop-seeds") + _explore.overrides.replay = config.getoption("--simloop-replay") + _explore.overrides.sim_tests = 0 + _explore.overrides.seeds_explored = 0 + + +def pytest_unconfigure(config: pytest.Config) -> None: + _explore.overrides.seeds = None + _explore.overrides.replay = None + _explore.overrides.node_id = None + + +def pytest_runtest_setup(item: pytest.Item) -> None: + _explore.overrides.node_id = item.nodeid + + +def pytest_runtest_teardown(item: pytest.Item) -> None: + _explore.overrides.node_id = None + + +def pytest_terminal_summary(terminalreporter: pytest.TerminalReporter) -> None: + tests = _explore.overrides.sim_tests + if not tests: + return + seeds = _explore.overrides.seeds_explored + noun = "sim test" if tests == 1 else "sim tests" + terminalreporter.write_line( + f"simloop: {tests} {noun}, {seeds:,} seeds explored" + ) diff --git a/tests/test_explore.py b/tests/test_explore.py new file mode 100644 index 0000000..568f74e --- /dev/null +++ b/tests/test_explore.py @@ -0,0 +1,202 @@ +"""Explorer core: first-failure seed search over fresh SimLoops.""" + +import asyncio +import subprocess +import sys + +import pytest + +import simloop +from simloop import SeedReport, sim_test +from simloop._explore import explore + + +async def _fails_at(bad_seed: int) -> None: + loop = asyncio.get_running_loop() + assert isinstance(loop, simloop.SimLoop) + await asyncio.sleep(1.0) + if loop.seed == bad_seed: + raise RuntimeError("boom") + + +def test_explore_reports_first_failing_seed() -> None: + report = explore(lambda: _fails_at(3), range(10)) + assert report is not None + assert report.seed == 3 + assert report.seeds_passed == 3 + assert isinstance(report.exception, RuntimeError) + assert str(report.exception) == "boom" + + +def test_explore_returns_none_when_all_seeds_pass() -> None: + assert explore(lambda: _fails_at(99), range(10)) is None + + +def test_explore_is_deterministic() -> None: + first = explore(lambda: _fails_at(7), range(10)) + second = explore(lambda: _fails_at(7), range(10)) + assert first is not None and second is not None + assert first.seed == second.seed + assert first.trace_hash == second.trace_hash + assert first.trace_events == second.trace_events + + +def test_trace_tail_is_bounded() -> None: + report = explore(lambda: _fails_at(0), range(1), trace_tail=5) + assert report is not None + assert len(report.trace_events) == 5 + assert report.trace_events[-1].kind in ("run", "cancel", "advance", "schedule", "net") + + +async def _interrupt() -> None: + raise KeyboardInterrupt + + +def test_base_exceptions_propagate() -> None: + with pytest.raises(KeyboardInterrupt): + explore(lambda: _interrupt(), range(3)) + + +async def _leaves_a_pending_task() -> None: + loop = asyncio.get_running_loop() + assert isinstance(loop, simloop.SimLoop) + loop.net.host("node1").create_task(_waits_forever(), name="stuck") + await asyncio.sleep(1.0) + raise RuntimeError("boom") + + +async def _waits_forever() -> None: + await asyncio.Event().wait() + + +def test_failed_run_leaves_no_stderr_noise( + capfd: pytest.CaptureFixture[str], +) -> None: + # A failed seed abandons its pending tasks; the explorer must tear them + # down so their garbage collection cannot write "Task was destroyed" + # to stderr after the run. + report = explore(lambda: _leaves_a_pending_task(), range(1)) + assert report is not None + import gc + + gc.collect() + _, err = capfd.readouterr() + assert err == "" + + +def test_render_includes_seed_trace_and_pending() -> None: + report = explore(lambda: _leaves_a_pending_task(), range(1), trace_tail=5) + assert report is not None + text = report.render("tests/test_demo.py::test_x") + lines = text.splitlines() + assert lines[0] == "simloop: failed at seed 0 (0 seeds passed first)" + assert lines[1] == ( + "replay: pytest 'tests/test_demo.py::test_x' --simloop-replay=0" + ) + assert "last 5 trace events:" in text + assert "pending tasks by host:" in text + assert "node1" in text and "'stuck'" in text + assert "awaiting _waits_forever" in text + + +def test_render_without_test_id_omits_replay_line() -> None: + report = explore(lambda: _fails_at(0), range(1)) + assert report is not None + text = report.render() + assert "replay:" not in text + assert text.startswith("simloop: failed at seed 0") + + +def test_sim_test_reraises_with_report_note() -> None: + @sim_test(seeds=10) + async def my_test() -> None: + await _fails_at(3) + + with pytest.raises(RuntimeError) as excinfo: + my_test() + notes = getattr(excinfo.value, "__notes__", []) + assert any("simloop: failed at seed 3" in note for note in notes) + + +def test_sim_test_passes_quietly() -> None: + @sim_test(seeds=5) + async def my_test() -> None: + await _fails_at(99) + + my_test() # must simply return + + +def test_sim_test_bare_form_defaults_to_ten_seeds() -> None: + ran: list[int] = [] + + @sim_test + async def my_test() -> None: + loop = asyncio.get_running_loop() + assert isinstance(loop, simloop.SimLoop) + ran.append(loop.seed) + + my_test() + assert ran == list(range(10)) + + +def test_sim_test_respects_replay_override() -> None: + from simloop._explore import overrides + + ran: list[int] = [] + + @sim_test(seeds=5) + async def my_test() -> None: + loop = asyncio.get_running_loop() + assert isinstance(loop, simloop.SimLoop) + ran.append(loop.seed) + + overrides.replay = 42 + try: + my_test() + finally: + overrides.replay = None + assert ran == [42] + + +def test_sim_test_respects_seed_count_override() -> None: + from simloop._explore import overrides + + ran: list[int] = [] + + @sim_test(seeds=2) + async def my_test() -> None: + loop = asyncio.get_running_loop() + assert isinstance(loop, simloop.SimLoop) + ran.append(loop.seed) + + overrides.seeds = 4 + try: + my_test() + finally: + overrides.seeds = None + assert ran == list(range(4)) + + +def test_sim_test_rejects_empty_seed_set() -> None: + with pytest.raises(ValueError): + + @sim_test(seeds=0) + async def my_test() -> None: + pass + + +def test_public_exports() -> None: + assert simloop.sim_test is sim_test + assert simloop.SeedReport is SeedReport + assert simloop.explore is explore + + +def test_import_simloop_does_not_import_pytest() -> None: + code = ( + "import simloop, sys; " + "raise SystemExit(1 if 'pytest' in sys.modules else 0)" + ) + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True + ) + assert proc.returncode == 0, proc.stderr diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py new file mode 100644 index 0000000..bb96d1a --- /dev/null +++ b/tests/test_pytest_plugin.py @@ -0,0 +1,113 @@ +"""Plugin behavior, exercised through real sub-pytest runs.""" + +import pytest + +_FLAKY = """ +import asyncio +from simloop import sim_test + + +@sim_test(seeds=10) +async def test_flaky(): + loop = asyncio.get_running_loop() + await asyncio.sleep(1.0) + assert loop.seed != 3 +""" + + +def test_failure_report_names_seed_and_replay_command( + pytester: pytest.Pytester, +) -> None: + pytester.makepyfile(test_demo=_FLAKY) + result = pytester.runpytest_subprocess() + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "*simloop: failed at seed 3 (3 seeds passed first)*", + "*replay: pytest 'test_demo.py::test_flaky' --simloop-replay=3*", + ] + ) + + +def test_replay_flag_runs_exactly_one_seed(pytester: pytest.Pytester) -> None: + pytester.makepyfile(test_demo=_FLAKY) + result = pytester.runpytest_subprocess("--simloop-replay=3") + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + ["*simloop: failed at seed 3 (0 seeds passed first)*"] + ) + result = pytester.runpytest_subprocess("--simloop-replay=4") + result.assert_outcomes(passed=1) + + +def test_seeds_flag_overrides_decorator_count(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + test_demo=""" +import asyncio +from simloop import sim_test + + +@sim_test(seeds=2) +async def test_flaky(): + loop = asyncio.get_running_loop() + await asyncio.sleep(1.0) + assert loop.seed != 3 +""" + ) + result = pytester.runpytest_subprocess() + result.assert_outcomes(passed=1) + result = pytester.runpytest_subprocess("--simloop-seeds=10") + result.assert_outcomes(failed=1) + + +def test_plugin_is_silent_without_sim_tests(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + test_demo=""" +def test_plain(): + assert True +""" + ) + result = pytester.runpytest_subprocess() + result.assert_outcomes(passed=1) + # Match "simloop:" (our output prefix), not bare "simloop" — pytest's + # own header prints "plugins: simloop-" for any installed + # entry-point plugin, and that must not fail this test. + result.stdout.no_fnmatch_line("*simloop:*") + + +def test_summary_counts_tests_and_seeds(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + test_demo=""" +import asyncio +from simloop import sim_test + + +@sim_test(seeds=7) +async def test_a(): + await asyncio.sleep(0.1) + + +@sim_test(seeds=5) +async def test_b(): + await asyncio.sleep(0.1) +""" + ) + result = pytester.runpytest_subprocess() + result.assert_outcomes(passed=2) + result.stdout.fnmatch_lines(["*simloop: 2 sim tests, 12 seeds explored*"]) + + +def test_summary_singular_for_one_test(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + test_demo=""" +import asyncio +from simloop import sim_test + + +@sim_test(seeds=3) +async def test_a(): + await asyncio.sleep(0.1) +""" + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*simloop: 1 sim test, 3 seeds explored*"]) diff --git a/uv.lock b/uv.lock index c1054e2..63d513a 100644 --- a/uv.lock +++ b/uv.lock @@ -235,7 +235,7 @@ wheels = [ [[package]] name = "simloop" -version = "0.0.1.dev0" +version = "0.1.0" source = { editable = "." } [package.dev-dependencies]