benchmarks: reusable registry, new model types, new phases, CI smoke

.github/dependabot.yml

@@ -12,3 +12,30 @@ updates:
     github-actions:
       patterns:
       - '*'
+
+# Pinned ``[benchmarks]`` extra in pyproject.toml. One PR per dep bump
+# → CodSpeed CI runs and attributes any perf delta to that specific
+# bump. Keeps the cross-version ``sweep`` baseline (lockfile-pinned)
+# stable while still surfacing upstream perf changes per-PR with
+# eyes-open review. Loose ``[project.dependencies]`` (numpy, scipy, ...)
+# have no version specifier so Dependabot leaves them alone — only the
+# ``==`` pins in ``[benchmarks]`` produce PRs.
+- package-ecosystem: pip
+  directory: /
+  schedule:
+    interval: monthly
+  open-pull-requests-limit: 5
+  groups:
+    # Measurement scaffolding + CLI/notebook tooling. Perf-irrelevant —
+    # they don't move CodSpeed signal, so batching into one PR cuts
+    # review noise. Perf-relevant deps (numpy, xarray, highspy, …) stay
+    # un-grouped so each gets its own attributed CodSpeed delta.
+    benchmark-tooling:
+      patterns:
+        - pytest
+        - pytest-benchmark
+        - pytest-memray
+        - pytest-codspeed
+        - nbconvert
+        - typer
+        - plotly

.github/workflows/benchmark-smoke.yml

@@ -0,0 +1,81 @@
+name: Benchmark smoke
+
+# Runs the internal benchmark suite under --quick --benchmark-disable so every
+# model spec is built and every phase fires at least once, but no timings are
+# recorded. The goal is "did refactor X break a model spec?" — not regression
+# tracking, which is done out-of-CI on dedicated hardware.
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '*' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    name: Benchmark smoke (quick)
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0  # setuptools_scm
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+
+    - name: Install package and benchmark dependencies
+      run: |
+        python -m pip install uv
+        # [dev] for pytest + netcdf4; [benchmarks] for pytest-benchmark + pypsa.
+        uv pip install --system -e ".[dev,benchmarks]"
+
+    - name: Run benchmark smoke
+      run: |
+        python -m benchmarks smoke
+
+    - name: Execute walkthrough notebook
+      # Catches doc rot — walkthrough.md is the canonical CLI walkthrough
+      # for the suite and must stay runnable end-to-end.
+      run: |
+        python -m benchmarks notebook
+
+  codspeed:
+    name: CodSpeed (micro regression detection)
+    runs-on: ubuntu-latest
+    # Cachegrind is ~10–20× slower than native, so we restrict to ``--quick``
+    # (smallest size per spec) and skip PyPSA end-to-end. The signal we want
+    # here is "did this PR change the instruction count of the hot paths?";
+    # full wall-clock cross-version comparison stays in ``sweep``.
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0  # setuptools_scm
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+
+    - name: Install pinned benchmark environment
+      # Install from the pinned ``[benchmarks]`` extra (not the lockfile)
+      # so Dependabot can auto-detect pyproject.toml and propose bumps
+      # to top-level deps. Each bump → one attributed CodSpeed delta.
+      # The full transitive lockfile is reserved for cross-version
+      # ``sweep`` reproducibility, where machine variance matters more.
+      run: |
+        python -m pip install uv
+        uv pip install --system -e ".[dev,benchmarks]"
+
+    - name: Run benchmarks under CodSpeed
+      uses: CodSpeedHQ/action@v3
+      with:
+        token: ${{ secrets.CODSPEED_TOKEN }}
+        run: |
+          pytest benchmarks/ --quick --codspeed

.gitignore

@@ -45,6 +45,12 @@ benchmark/scripts/__pycache__
 benchmark/scripts/benchmarks-pypsa-eur/__pycache__
 benchmark/scripts/leftovers/
 
+# Benchmarks (internal suite in benchmarks/) — the .md walkthrough is
+# canonical; ``python -m benchmarks notebook --build`` regenerates the
+# .ipynb sibling as a throwaway viewing/running artifact.
+benchmarks/walkthrough.ipynb
+benchmarks/.ipynb_checkpoints/
+
 # IDE
 .idea/
 

benchmarks/README.md

@@ -1,94 +1,44 @@
 # Internal Performance Benchmarks
 
-Measures linopy's own performance (build time, LP write speed, memory usage) across problem sizes using [pytest-benchmark](https://pytest-benchmark.readthedocs.io/) and [pytest-memray](https://pytest-memray.readthedocs.io/). Use these to check whether a code change introduces a regression or improvement.
+End-to-end performance tracking for `linopy` — build → solver handoff
+→ netCDF (de)serialization → fixed PyPSA model. Solver algorithm
+runtime is out of scope.
 
-> **Note:** The `benchmark/` directory (singular) contains *external* benchmarks comparing linopy against other modeling frameworks. This directory (`benchmarks/`) is for *internal* performance tracking only.
+**The walkthrough is load-bearing.** Phase coverage, CLI introspection,
+the two-snapshot regression workflow with inline Plotly views, and
+how to extend the suite live in [`walkthrough.md`](walkthrough.md).
+This README only covers install and how to open the walkthrough.
 
-## Setup
+> `benchmark/` (singular) is the legacy external-framework suite.
+> `benchmarks/` (plural) is this internal suite.
 
-```bash
-pip install -e ".[benchmarks]"
-```
-
-## Running benchmarks
+## Install
 
 ```bash
-# Quick smoke test (small sizes only)
-pytest benchmarks/ --quick
-
-# Full timing benchmarks
-pytest benchmarks/test_build.py benchmarks/test_lp_write.py benchmarks/test_matrices.py
-
-# Run a specific model
-pytest benchmarks/test_build.py -k basic
-```
-
-## Comparing timing between branches
-
-```bash
-# Save baseline results on master
-git checkout master
-pytest benchmarks/test_build.py --benchmark-save=master
-
-# Switch to feature branch and compare
-git checkout my-feature
-pytest benchmarks/test_build.py --benchmark-save=my-feature --benchmark-compare=0001_master
-
-# Compare saved results without re-running
-pytest-benchmark compare 0001_master 0002_my-feature --columns=median,iqr
+uv sync --extra dev --extra benchmarks
+source .venv/bin/activate
 ```
 
-Results are stored in `.benchmarks/` (gitignored).
+`pypsa` is optional — `pypsa_scigrid` and
+`test_pypsa_carbon_management.py` skip gracefully without it. Install
+when you need them: `uv pip install pypsa`.
 
-## Memory benchmarks
+The `[benchmarks]` extra in `pyproject.toml` pins every direct dep that
+affects measurement (`numpy`, `scipy`, `xarray`, `pandas`, `polars`,
+`dask`, etc.). `sweep` installs these into each per-version venv, so
+"same deps, only linopy varies" comes for free without a separate
+lockfile — bump the pins in pyproject and the next sweep picks them up.
 
-`memory.py` runs each test in a separate process with pytest-memray to get accurate per-test peak memory (including C/numpy allocations). Results are saved as JSON and can be compared across branches.
-
-By default, only the build phase (`test_build.py`) is measured. Unlike timing benchmarks where `benchmark()` isolates the measured function, memray tracks all allocations within a test — including model construction in setup. This means LP write and matrix tests would report build + phase memory combined, making the phase-specific contribution impossible to isolate. Since model construction dominates memory usage, measuring build alone gives the most actionable numbers.
+## Open the walkthrough
 
 ```bash
-# Save baseline on master
-git checkout master
-python benchmarks/memory.py save master
-
-# Save feature branch
-git checkout my-feature
-python benchmarks/memory.py save my-feature
-
-# Compare
-python benchmarks/memory.py compare master my-feature
-
-# Quick mode (smaller sizes, faster)
-python benchmarks/memory.py save master --quick
-
-# Measure a specific phase (includes build overhead)
-python benchmarks/memory.py save master --test-path benchmarks/test_lp_write.py
+python -m benchmarks notebook --build       # (re)generate walkthrough.ipynb
+jupyter lab benchmarks/walkthrough.ipynb    # ...or PyCharm / VSCode
 ```
 
-Results are stored in `.benchmarks/memory/` (gitignored). Requires Linux or macOS (memray is not available on Windows).
-
-> **Note:** Small tests (~5 MiB) are near the import-overhead floor and may show noise of ~1 MiB between runs. Focus on larger tests for meaningful memory comparisons. Do not combine `--memray` with timing benchmarks — memray adds ~2x overhead that invalidates timing results.
-
-## Models
-
-| Model | Description | Sizes |
-|-------|-------------|-------|
-| `basic` | Dense N*N model, 2*N^2 vars/cons | 10 — 1600 |
-| `knapsack` | N binary variables, 1 constraint | 100 — 1M |
-| `expression_arithmetic` | Broadcasting, scaling, summation across dims | 10 — 1000 |
-| `sparse_network` | Ring network with mismatched bus/line coords | 10 — 1000 |
-| `pypsa_scigrid` | Real power system (requires `pypsa`) | 10 — 200 snapshots |
-
-## Phases
-
-| Phase | File | What it measures |
-|-------|------|------------------|
-| Build | `test_build.py` | Model construction (add_variables, add_constraints, add_objective) |
-| LP write | `test_lp_write.py` | Writing the model to an LP file |
-| Matrices | `test_matrices.py` | Generating sparse matrices (A, b, c, bounds) from the model |
-
-## Adding a new model
+The `.md` is the source of truth; the `.ipynb` is a disposable,
+gitignored build artifact. Edit the `.md`, re-run `--build`, re-open.
+Same workflow in any editor.
 
-1. Create `benchmarks/models/my_model.py` with a `build_my_model(n)` function and a `SIZES` list
-2. Add parametrized tests in the relevant `test_*.py` files
-3. Add a quick threshold in `conftest.py`
+CI executes the walkthrough end-to-end on every PR
+(`python -m benchmarks notebook`) so the examples can't silently rot.

benchmarks/__init__.py

@@ -1 +1,114 @@
-"""Linopy benchmark suite — run with ``pytest benchmarks/`` (use ``--quick`` for smaller sizes)."""
+"""
+Linopy benchmark suite.
+
+Run with ``pytest benchmarks/`` (use ``--quick`` for smaller sizes).
+
+This package also exposes a **reusable model registry** for any test, profiling
+session, or example that wants ready-made linopy models of varying sizes and
+features. Each entry exposes a ``build(size) -> linopy.Model`` callable plus
+metadata::
+
+    from benchmarks import REGISTRY, QUADRATIC
+
+    # Look up by name
+    model = REGISTRY["basic"].build(100)
+
+    # Iterate / filter
+    for spec in REGISTRY.values():
+        m = spec.build(spec.sizes[0])
+        ...
+
+    from benchmarks import filter_by
+    qp_specs = filter_by(has_feature=QUADRATIC)
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    import pandas as pd
+
+    from benchmarks.snapshot import Metric
+
+# Importing the models package triggers each module's ``register(...)`` call.
+from benchmarks import bench, models  # noqa: F401, E402
+
+
+def load_long_df(
+    snapshots: list[Path], metric: Metric = "min"
+) -> tuple[pd.DataFrame, str]:
+    """
+    Load one or more benchmark JSON snapshots into a tidy DataFrame.
+
+    Thin re-export of :func:`benchmarks.snapshot.load_long_df` so callers
+    can do their own analysis without importing the plotting module
+    (which pulls in plotly). Returns ``(df, unit)`` where ``df`` has one
+    row per ``(snapshot, test_id)`` with columns ``snapshot, test_id,
+    phase, model, size, value``, and ``unit`` is ``"s"`` (timing) or
+    ``"MiB"`` (memory).
+    """
+    from benchmarks.snapshot import load_long_df as _impl
+
+    return _impl(snapshots, metric)
+
+
+from benchmarks.registry import (  # noqa: F401, E402 — re-export
+    ALL_FEATURES,
+    ALL_PHASES,
+    BINARY,
+    BUILD,
+    CONTINUOUS,
+    DEFAULT_PHASES,
+    INTEGER,
+    LP_WRITE,
+    MASKED,
+    MATRICES,
+    NETCDF,
+    PIECEWISE,
+    QUADRATIC,
+    REGISTRY,
+    SOS,
+    TO_GUROBIPY,
+    TO_HIGHSPY,
+    TO_MOSEK,
+    TO_XPRESS,
+    ModelSpec,
+    filter_by,
+    get,
+    iter_params,
+    param_ids,
+    register,
+)
+
+__all__ = [
+    "ALL_FEATURES",
+    "ALL_PHASES",
+    "BINARY",
+    "BUILD",
+    "CONTINUOUS",
+    "DEFAULT_PHASES",
+    "INTEGER",
+    "LP_WRITE",
+    "MASKED",
+    "MATRICES",
+    "ModelSpec",
+    "NETCDF",
+    "PIECEWISE",
+    "QUADRATIC",
+    "REGISTRY",
+    "SOS",
+    "TO_GUROBIPY",
+    "TO_HIGHSPY",
+    "TO_MOSEK",
+    "TO_XPRESS",
+    "bench",
+    "filter_by",
+    "get",
+    "iter_params",
+    "load_long_df",
+    "param_ids",
+    "register",
+]

benchmarks/__main__.py

@@ -0,0 +1,5 @@
+"""Allow ``python -m benchmarks <command>``."""
+
+from benchmarks.cli import app
+
+app()