refactor: consume psyexp-core for shared experiment infrastructure

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       # Guard the release: the tag, pyproject.toml version, and CHANGELOG.md must
       # agree. Fail loudly rather than publish a mislabeled release.
@@ -48,7 +48,7 @@ jobs:
       # `uv lock --check` fails if the lock is out of date (e.g. the version was
       # bumped in pyproject.toml but uv.lock was never re-locked to match).
       - name: Verify uv.lock is in sync with pyproject.toml
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
       - name: uv lock --check
         run: uv lock --check
 
@@ -79,7 +79,7 @@ jobs:
           echo "file=$notes_file" >> "$GITHUB_OUTPUT"
 
       - name: Create GitHub Release
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
         with:
           name: ${{ github.ref_name }}
           body_path: ${{ steps.notes.outputs.file }}

.github/workflows/tests.yml

@@ -8,7 +8,7 @@ jobs:
   pytest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       # Octave is the reference engine for the calibration MATLAB-parity test
       # (tests/test_calibration_matlab_parity.py). Without it that test skips
@@ -21,18 +21,29 @@ jobs:
           octave --version | head -1
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
 
       # PsychoPy is deliberately NOT installed: it's a heavy GUI/audio dep whose
       # transitive build (ffpyplayer → SDL headers) is fragile on headless CI.
       # The source modules guard their psychopy imports (try/except), so the
-      # whole test suite imports and runs without it; only pandas is needed at
-      # runtime (sequences.py). We install just that + pytest and put `src` on
-      # PYTHONPATH rather than resolving the project's full dependency graph.
+      # whole test suite imports and runs without it; pandas (sequences.py) and
+      # pydantic (psyexp-core's manifest models, since core 0.7.0) are needed at
+      # runtime. We install just those + pytest and put `src` on PYTHONPATH
+      # rather than resolving the project's full dependency graph.
+      #
+      # psyexp-core is installed --no-deps for the same reason: the light modules
+      # the tests pull in (recording / manifest / rundir / diagnostics / keyboard)
+      # are import-clean without PsychoPy, so --no-deps avoids dragging the GUI
+      # stack back in (pydantic, installed above, is its one non-GUI runtime dep
+      # the tests touch). Its pinned version is read from uv.lock via `uv export`
+      # rather than hardcoding it here — the lockfile stays the single source of
+      # truth.
       - name: Create venv and install minimal test deps
         run: |
           uv venv
-          uv pip install pytest pandas
+          uv pip install pytest pandas pydantic
+          uv export --frozen --no-hashes --no-emit-project | grep '^psyexp-core==' > psyexp-core-req.txt
+          uv pip install --no-deps -r psyexp-core-req.txt
 
       # tests/test_overlay.py is excluded: it's not an automated test (no test
       # functions) but a manual `visual.Window` script that needs a live display

AGENTS.md

@@ -19,4 +19,44 @@ engine the test skips with a loud warning rather than silently passing.
 and its PyPI dependencies from `pyproject.toml` (`pip install -e .`). The UV-only sections
 (`[tool.uv.*]`) and `uv.lock` are silently ignored by pip/conda, so no `pyproject.toml` changes are
 needed to support conda — conda just resolves dependencies fresh from PyPI instead of from the
-lockfile.
+lockfile.
+
+## Shared harness: psyexp-core
+
+The generic experiment plumbing — fullscreen window + VSYNC/frame-timing setup, timestamped run
+directories, the `CsvWriter` base, run-manifest writing, setup-wizard primitives, the instruction
+pager, and the keyboard abstraction — lives in the shared [`psyexp-core`](https://github.com/HAPNlab/psyexp-core)
+package. This repo keeps only MID-specific logic (trial loop, reward rule, adaptive staircase,
+sequences, ratings survey, legacy MATLAB CSV) and consumes the harness as a dependency.
+
+`psyexp-core` is a **published PyPI package**, declared in `[project].dependencies` as
+`psyexp-core>=X.Y`; the exact version is pinned in `uv.lock` so both uv and pip/conda resolve the
+same core, and every run's `manifest.json` records the resolved `psyexp_core_version`.
+
+To **co-develop** the core against this task, overlay an editable sibling checkout. The catch:
+`uv.lock` is authoritative, so *any* `uv sync` reverts the overlay back to the locked PyPI version —
+`--inexact` does **not** help (it only spares packages absent from the lock, and the core is in it).
+The one thing that preserves the overlay is skipping the sync:
+
+```sh
+uv pip install -e ../psyexp-core
+uv run --no-sync pytest        # or: export UV_NO_SYNC=1 for the shell
+```
+
+The `just` recipes wrap this: `just core-dev` overlays the editable checkout, then `just core-run` /
+`just core-test` run with `--no-sync`; `just core-release` drops it (plain `uv sync`). Don't run a
+bare `uv sync`/`uv run` while overlaying — it silently reverts the editable core. For a setup that
+survives sync, declare the path source in `pyproject.toml`
+(`[tool.uv.sources] psyexp-core = { path = "../psyexp-core", editable = true }`) and keep that edit
+local with `git update-index --skip-worktree pyproject.toml uv.lock` so it never lands in a commit.
+
+**Updating to a new `psyexp-core` release:** a bare `uv sync` won't move it — it installs exactly
+what `uv.lock` pins. Re-resolve the lock, then apply it (or run `just core-upgrade`):
+
+```sh
+uv lock --upgrade-package psyexp-core   # rewrite uv.lock to the newest version the constraint allows
+uv sync
+```
+
+Commit the updated `uv.lock` (and bump the `>=` floor in `pyproject.toml` first if you want to
+require a new minimum).

docs/STANDARDIZATION.md

@@ -0,0 +1,93 @@
+# Standardizing task design across PsychoPy tasks
+
+Status: **in progress**. This tracks where `mid-task-deterministic` and
+[`heat-task`](https://github.com/HAPNlab/heat-task) still diverge and what belongs in
+[`psyexp-core`](https://github.com/HAPNlab/psyexp-core) versus each task repo. The
+consumer-side work in this repo (standard keymap schema, consuming the core
+keyboard helpers, local `ExitStack` teardown) is **done** and lives in the code;
+what remains below is cross-repo coordination and deliberately deferred work.
+
+Guiding rule: **task repos keep only task-specific logic; anything a second task
+would copy belongs in `psyexp-core`.**
+
+---
+
+## Open: align heat-task on the core manifest API
+
+`psyexp-core` renamed `write_manifest(session_time=…)` →
+`write_manifest(session_started_at=…)` in **0.8.0**. mid-task moved to 0.8.0 and
+the new kwarg; **heat-task is still on 0.7.0 and still passes `session_time=`**.
+So the two repos are pinned to different core majors and call the manifest writer
+differently.
+
+**Action:** bump heat-task to `psyexp-core>=0.8.0` and rename its `write_manifest`
+call's kwarg to `session_started_at`; then bump to `0.9.0` and replace its
+`task/phases.py` `wait_for_key`/`check_quit` with the core ones (mid-task already
+does this). Keep the per-run `psyexp_core_version` in every manifest so the
+resolved core is always recoverable. Treat the core's public signature as a shared
+contract: a rename is a coordinated bump across all task repos in the same batch.
+
+## Deferred: candidate core extractions
+
+Worth centralizing eventually, but not until a second consumer makes the dedup pay
+for itself:
+
+- **§ Live-view console → `psyexp_core.console.LiveTableView`.**
+  `mid_det/task/console.py::TrialLiveView` and
+  `heat_task/task/console.py::SequenceLiveView` share a skeleton (a Rich `Live`
+  context manager, a `_RowData` dataclass, `_make_table`, throttled `_refresh`).
+  A base class would own the `Live` lifecycle and `_refresh(force=…)` throttling;
+  subclasses declare columns and row rendering, with heat's status line/blink as an
+  optional mixin. Low priority — the two views differ enough that the shared
+  surface is mostly the container.
+
+- **§ Teardown → shared `psyexp_core.experiment_session`.** Both tasks now wrap
+  `run()` teardown in a **local** `contextlib.ExitStack` (heat derives MMS
+  stop-vs-abort from `exc_type`; mid gained guaranteed teardown it previously
+  lacked). Each calls `core.quit()` *after* the block so a genuine error still
+  surfaces its traceback. Because `ExitStack.__exit__` fires on `BaseException`,
+  Ctrl-C and the `SystemExit` from `core.quit()` both trigger teardown for free.
+  Promoting this to a shared helper stays deferred until a third — or second
+  hardware — task makes it worthwhile.
+
+- **§ Operator-gated screens → core.** mid's `wait_for_start` / `run_end_screen`
+  become generic only once they take a "draw this frame" callback (the inversion
+  `psyexp_core.instructions.page_through` already uses). Separate extraction.
+
+## Rejected: a `psyexp_core.runner` scaffold
+
+Both `__main__.py::run()` functions walk a similar spine (backend → screen →
+wizard → run dir + logging → stimuli → manifest → instructions → loop → end →
+cleanup), which tempts a "extract one `run()` scaffold and pass hooks" design.
+**Don't** — the similarities are cosmetic and the differences are load-bearing:
+
+- **Ordering inverts.** mid opens the screen *before* the wizard (the wizard needs
+  the measured frame duration for its RT-field defaults); heat runs the wizard
+  *first* (it returns `screen_index` and must precede the window). A scaffold with
+  fixed `wizard`/`screen` slots is wrong for one task no matter which order it picks.
+- **Teardown is asymmetric.** heat chooses MMS `abort` vs `stop` on the way out (a
+  thermode left running is a safety issue); mid just closes CSV writers.
+- **The middles are genuinely task-specific** (scanner pulse-counter + adaptive
+  calibration + flip-patch vs. background `StatusPoller` + MMS program-select) and
+  interleave differently.
+
+So `run()` stays per-task: an explicit top-to-bottom script you can follow beats
+indirection through a framework, and the reuse comes from shared *leaf* helpers
+(`screen.setup_screen`, a frame-rate resolver, `rundir.make_run_dir`,
+`write_manifest`, `wait_for_key`, the eventual `LiveTableView`), not an
+orchestrator. *Library, not framework.* What stays duplicated is ~15 lines of glue
+— the order the helpers are called in — not logic.
+
+## Left as-is: hand-rolling is correct
+
+Considered replacing with a library, but the hand-rolled version is the right call:
+
+- **`StatusPoller` exponential backoff (heat `task/status.py`) → tenacity/backoff?**
+  No. The bounded backoff is ~3 lines and is coupled to `stop_event.wait()` for
+  prompt cancellation; retry libraries wrap a *callable* and don't fit a
+  long-lived, stop-event-interruptible reconnect-and-stream loop.
+- **`sum(x) / len(x)` rolling means (heat `phase_tracker.py`) → numpy?** No. They
+  run per-sample over ~5-element windows; building an ndarray each sample is slower.
+  numpy is correctly reserved for the calibration array.
+- **Plain `@dataclass` CSV-row records → pydantic?** No. Internal records with no
+  validation need; pydantic would be pure overhead.

docs/development.md

@@ -61,6 +61,47 @@ pip install -e ".[dev]"
 `uv.lock`) are ignored by pip/conda, so the conda install resolves dependencies fresh from PyPI
 rather than from the lockfile.
 
+## Updating `psyexp-core`
+
+The shared experiment harness lives in the published [`psyexp-core`](https://github.com/HAPNlab/psyexp-core)
+package, declared as `psyexp-core>=X.Y` in `pyproject.toml` with the exact version pinned in
+`uv.lock`. A bare `uv sync` does **not** pull a newer release — it installs exactly what `uv.lock`
+pins, so a newly published version is ignored until the lock is regenerated. To upgrade:
+
+```bash
+uv lock --upgrade-package psyexp-core   # rewrite uv.lock to the newest version the constraint allows
+uv sync                                 # apply it
+```
+
+Then commit the updated `uv.lock`. Raise the `>=` floor in `pyproject.toml` first if you want to
+require a new minimum. (`just core-upgrade` runs both commands.)
+
+## Co-developing `psyexp-core` locally
+
+To work on the harness from a sibling checkout, overlay an editable install. The catch: `uv.lock`
+is authoritative, so **any `uv sync` reverts the overlay** back to the locked PyPI version.
+`--inexact` does *not* help — it only retains packages absent from the lock, and the core is in it.
+The only thing that preserves the overlay is skipping the sync entirely:
+
+```bash
+uv pip install -e ../psyexp-core    # one time
+uv run --no-sync mid-task-det       # or: export UV_NO_SYNC=1 for the shell
+```
+
+The `just` recipes wrap this workflow:
+
+```bash
+just core-dev        # overlay the editable ../psyexp-core
+just core-run        # run the task with --no-sync (overlay preserved)
+just core-test       # run pytest with --no-sync
+just core-release    # drop the overlay, restore the locked PyPI core (plain uv sync)
+```
+
+Don't run a bare `uv sync`/`uv run` while overlaying — it silently reverts the editable core. For a
+setup that survives sync, declare the path source in `pyproject.toml`
+(`[tool.uv.sources] psyexp-core = { path = "../psyexp-core", editable = true }`) and keep that edit
+local with `git update-index --skip-worktree pyproject.toml uv.lock` so it never lands in a commit.
+
 ## Project Structure
 
 ```
@@ -97,7 +138,7 @@ mid-task-deterministic/
 │   ├── run_2.csv             # 54-trial sequence for run 2
 │   └── practice.csv          # 18-trial practice (one trial per condition)
 ├── text/
-│   └── instructions_MID.txt  # Instruction pages (one line per page)
+│   └── instructions_ratings.txt  # Cue-ratings instructions (MID task pages are hardcoded in task/instructions.py)
 ├── data/                     # Output directory (created at runtime)
 ├── tests/
 ├── docs/

justfile

@@ -0,0 +1,50 @@
+# mid-task-deterministic dev tasks. Run `just` to list recipes.
+# Requires `just` (https://github.com/casey/just): `brew install just`.
+
+# Show available recipes
+default:
+    @just --list
+
+# Install/refresh the venv from the locked PyPI psyexp-core
+sync:
+    uv sync
+
+# Run the experiment (locked PyPI core)
+run *args:
+    uv run mid-task-det {{args}}
+
+# Run the standalone cue-ratings survey
+ratings *args:
+    uv run mid-ratings-det {{args}}
+
+# Run the test suite
+test *args:
+    uv run pytest {{args}}
+
+# --- Co-developing psyexp-core from ../psyexp-core ---------------------------
+# uv.lock is authoritative, so a plain `uv sync`/`uv run` reverts an editable
+# overlay back to the locked PyPI version. `--inexact` does NOT prevent this — it
+# only spares packages absent from the lock, and the core is in it. The only way
+# to keep the overlay is to skip the sync, which these recipes do via --no-sync.
+
+# Overlay an editable sibling checkout of ../psyexp-core
+core-dev:
+    uv pip install -e ../psyexp-core
+    @echo "Editable psyexp-core overlaid. Use 'just core-run' / 'just core-test' so it isn't reverted."
+
+# Run the experiment against the editable overlay (skips the revert-causing sync)
+core-run *args:
+    uv run --no-sync mid-task-det {{args}}
+
+# Run the tests against the editable overlay
+core-test *args:
+    uv run --no-sync pytest {{args}}
+
+# Drop the overlay and restore the locked PyPI psyexp-core
+core-release:
+    uv sync
+
+# Upgrade to the newest published psyexp-core and update the lock
+core-upgrade:
+    uv lock --upgrade-package psyexp-core
+    uv sync

pyproject.toml

@@ -7,9 +7,19 @@ name = "mid-task-deterministic"
 version = "1.0.0-rc.2"
 description = "Deterministic Monetary Incentive Delay Task - PsychoPy implementation for fMRI (no adaptive staircase)"
 readme = "README.md"
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.12"
 authors = [{ name = "Eric Wang" }]
 dependencies = [
+    # Shared experiment harness (screen/frame-timing, run dirs, manifests, CSV
+    # writers, setup-wizard primitives, instruction pager, keyboard abstraction).
+    # Published PyPI package; the exact version is pinned in uv.lock for
+    # reproducible clones and the run manifest also records psyexp_core_version.
+    # pip/conda resolve it too (production uses conda — see AGENTS.md). For
+    # co-development, overlay an editable sibling checkout and run with --no-sync
+    # (any uv sync reverts it — --inexact does NOT help; the core is in the lock):
+    #   uv pip install -e ../psyexp-core
+    #   uv run --no-sync ...        # or: export UV_NO_SYNC=1  (see `just core-*`)
+    "psyexp-core>=0.8.0",
     "psychopy>=2026.1",
     "scipy>=1.11",
     "numpy>=1.26",