Add deterministic multi-worker evaluation and seeded failure replay

.github/workflows/ci.yml

@@ -40,6 +40,10 @@ jobs:
       - name: Unit tests
         run: python -m pytest -v tests/unit_tests
 
+      - name: Evaluation integration tests
+        run: python -m pytest -v tests/eval
+        timeout-minutes: 15
+
   c-drive-tests:
     name: C Drive Tests
     runs-on: ubuntu-latest
@@ -121,40 +125,6 @@ jobs:
         run: docker run --rm pufferdrive-smoke
         timeout-minutes: 30
 
-  smoke-replay:
-    name: Smoke Replay HTML
-    runs-on: ubuntu-latest
-    env:
-      PUFFER_CPU: 1
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - name: Free up disk space
-        run: |
-          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
-          sudo apt-get clean
-          sudo rm -rf ~/.cache/pip /tmp/* /var/tmp/*
-
-      - name: Install pufferlib
-        env:
-          TMPDIR: ${{ runner.temp }}/build
-          PIP_NO_CACHE_DIR: 1
-        run: |
-          sudo apt-get update && sudo apt-get install -y build-essential cmake
-          python -m pip install -U pip pytest
-          pip install -e . --no-cache-dir
-          python setup.py build_ext --inplace --force
-
-      - name: Replay HTML test
-        run: python -m pytest -v tests/smoke_tests/test_validation_replay_html.py
-        timeout-minutes: 20
-
   perf-tests:
     name: Smoke Performance Test
     runs-on: ubuntu-latest

MANIFEST.in

@@ -4,6 +4,7 @@ global-include *.h
 global-include *.cpp
 global-include *.cu
 global-include *.py
+recursive-include pufferlib/config *.ini *.yaml
 recursive-include pufferlib/resources *
 recursive-exclude experiments *
 recursive-exclude wandb *

Makefile

@@ -10,30 +10,30 @@ EXT_SUFFIX := $(shell $(PYTHON) -c "import sysconfig; print(sysconfig.get_config
 BINDING := pufferlib/ocean/drive/binding$(EXT_SUFFIX)
 DRIVE_SOURCES := $(wildcard pufferlib/ocean/drive/*.c) $(wildcard pufferlib/ocean/drive/*.h) setup.py
 
-.PHONY: help test rebuild ensure-test-deps test-unit test-c test-smoke test-notebooks test-docker-smoke
+.PHONY: help test rebuild ensure-test-deps test-unit test-eval test-c test-notebooks test-docker-smoke
 
 help:
 	@echo "PufferDrive test targets:"
 	@echo ""
-	@echo "  make test               Run the local suites (unit + C + smoke + notebooks),"
+	@echo "  make test               Run the local suites (unit + eval + C + notebooks),"
 	@echo "                          fail-fast. Rebuilds the C extension first if any"
 	@echo "                          .c/.h changed; installs test deps if missing."
 	@echo "  make rebuild            Force-rebuild the C extension unconditionally"
 	@echo ""
 	@echo "  make test-unit          Python unit tests (tests/unit_tests)"
+	@echo "  make test-eval          Evaluation integration tests (tests/eval)"
 	@echo "  make test-c             C sim tests (tests/drive: dynamics, geometry, IDM, ...)"
-	@echo "  make test-smoke         Replay-HTML smoke test"
 	@echo "  make test-notebooks     Execute every checked-in notebook end-to-end"
 	@echo ""
-	@echo "  make test-docker-smoke  Full smoke suite in Docker (train/rollout/eval goldens)"
+	@echo "  make test-docker-smoke  Full smoke suite in Docker (train/rollout goldens)"
 
 $(BINDING): $(DRIVE_SOURCES)
 	$(PYTHON) setup.py build_ext --inplace --force
 
-# Local suites, fast-failing first: Python unit tests, C sim tests,
-# replay-HTML smoke test, then the notebook executions (slowest). The
+# Local suites, fast-failing first: Python unit tests, eval integration, C sim tests,
+# then the notebook executions (slowest). The
 # Docker smoke suite stays opt-in — it needs a Docker daemon.
-test: test-unit test-c test-smoke test-notebooks
+test: test-unit test-eval test-c test-notebooks
 
 rebuild:
 	$(PYTHON) setup.py build_ext --inplace --force
@@ -46,12 +46,12 @@ ensure-test-deps:
 test-unit: $(BINDING)
 	$(PYTHON) -m pytest -v tests/unit_tests
 
+test-eval: $(BINDING)
+	$(PYTHON) -m pytest -v tests/eval
+
 test-c:
 	$(MAKE) -C tests/drive test
 
-test-smoke: $(BINDING)
-	$(PYTHON) -m pytest -v tests/smoke_tests/test_validation_replay_html.py
-
 test-notebooks: $(BINDING) ensure-test-deps
 	$(PYTHON) -m pytest -v tests/notebooks
 

README.md

@@ -131,67 +131,42 @@ torchrun --standalone --nnodes=1 --nproc-per-node=6 -m pufferlib.pufferl train p
 
 ## Eval
 
-All evaluation runs through the unified `Evaluator`/`EvalManager` pipeline.
-`eval.<name>` sections in `puffer_drive.yaml` define each evaluator; the same
-ones run inline during training and standalone here.
-
-The default device is CUDA; on a machine without it (e.g. a Mac) add `train.device=cpu`.
+The eval command loads one or more named benchmarks from the benchmark YAML and
+writes per-episode metrics for each benchmark. Benchmark selection is mandatory;
+eval does not choose one implicitly. The default device is CUDA; on a machine
+without it add `train.device=cpu`.
 
 ```bash
-# Run a named evaluator on a checkpoint (config from eval.<name>)
-puffer eval puffer_drive --evaluator validation_gigaflow \
-  load_model_path=experiments/puffer_drive_xxxx/models/model_puffer_drive_000500.pt
-
-# Ad-hoc: pick by simulation + override scale from the CLI
-puffer eval puffer_drive --eval_simulation replay \
-  load_model_path=experiments/puffer_drive_xxxx/models/model_puffer_drive_000500.pt \
-  --num_scenarios 250 --render 1
-
-# Render the agent's observations (interactive HTML)
-puffer eval puffer_drive --eval_simulation gigaflow \
+# Run the CARLA YAML benchmark and render every evaluated scenario
+puffer eval puffer_drive carla \
   load_model_path=experiments/puffer_drive_xxxx/models/model_puffer_drive_000500.pt \
-  --num_scenarios 10 --render 1 --render-backend obs_html
+  eval.render_scenarios=true \
+  eval.capture_observations=false
 ```
 
-(`--evaluator`, `--eval_simulation`, `--num_scenarios`, `--render`, and
-`--render-backend` are per-invocation eval flags consumed before config
-loading; everything else uses Hydra `key=value` overrides.)
-
-**For the full guide see [`docs/evaluation.md`](docs/evaluation.md).**
-
-## Failure mining
-
-Roll a trained policy out against a scenario suite, capture per-episode compact replays for episodes whose `episode_return` falls below a threshold, render each one as an interactive HTML page, and produce a sortable cross-episode index. Useful for triaging what a policy fails at after a long training run.
+Scenario renders are captured during the benchmark pass and written as
+interactive HTML with retained `.replay.zlib` files. To render only episodes
+where `offroad_rate > 0` instead:
 
 ```bash
-puffer mine_failures puffer_drive \
-    load_model_path=experiments/puffer_drive_xxxx/models/model_puffer_drive_000123.pt \
-    mine.output_dir=./failure_mining/puffer_drive_xxxx \
-    mine.num_episodes=200 \
-    mine.score_threshold=-10.0
-```
-
-Config keys (under `mine:` in `puffer_drive.yaml` or `mine.<key>=<value>` on the CLI):
-
-| Key | Default | Notes |
-|---|---|---|
-| `output_dir` | `./failure_mining/<env_name>` | Where replays, CSV, and HTML output go |
-| `num_episodes` | `100` | Total episodes to roll out |
-| `score_threshold` | `-inf` | `episode_return < threshold` → flagged as failure; replay written to disk. With `-inf`, no failures are flagged and no replays are persisted. |
-| `render` | `True` | Render each captured replay to HTML + write `index.html` via `mining_viz` |
-
-`env.*` overrides apply (e.g. `env.simulation_mode=gigaflow` to mine on procedural scenarios). Single vec env, sequential rollout — no per-worker map pinning yet. On a machine without CUDA add `train.device=cpu vec.backend=Serial` (the default vec config assumes cluster core counts).
-
-**Output structure** (under `output_dir`):
-
-```
-episodes.csv                 # one row per episode, all summary metrics
-replays/episode_NNNNNN.replay.zlib   # only for failures (above-threshold episodes are not persisted)
-renders/episode_NNNNNN.html  # one viewer page per failure
-renders/index.html           # sortable index of all episodes
+puffer eval puffer_drive carla \
+  load_model_path=experiments/puffer_drive_xxxx/models/model_puffer_drive_000500.pt \
+  eval.render_filter=offroad_rate \
+  eval.max_rendered_failures=10 \
+  eval.capture_observations=false
 ```
 
-Open `renders/index.html` in a browser to triage. The index page filters by "failures only" / "replays only" and sorts by any metric column. Each row links to the per-episode viewer with the scene's full 2D animation.
+Use `eval.num_agents`, not `env.num_agents`, to configure evaluation capacity.
+Evaluation outputs are written under
+`eval/<benchmark>[_<output_name>]/<timestamp>/`. The benchmark seed and effective
+worker count are saved in the
+resolved configuration: repeated runs with both unchanged produce the same
+map/seed rows. Filtered renders replay the exact map and seed recorded by the
+metrics pass. Set `eval.failure_replay_csv=<path/to/episode_metrics.csv>` with a
+non-null `eval.render_filter` to skip the standard benchmark pass and filter an
+existing report directly. See [docs/evaluation.md](docs/evaluation.md) for the
+benchmark schema, filtered replay and all-scenario rendering flows, outputs, and
+training integration.
 
 ## Nightly runs and the regression report