Add action to run benchmarks

.github/workflows/cargo-test.yml

@@ -40,3 +40,53 @@ jobs:
         uses: ./.github/actions/codecov
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+
+  benchmark:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: write # needed to push the updated baseline back to the repo
+      pull-requests: write # needed to post a comment when an alert is triggered
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Run cargo bench
+        shell: bash
+        run: cargo bench --features bench --no-fail-fast
+
+      - name: Convert Criterion results to JSON
+        run: python3 benches/convert_criterion_output.py
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "customSmallerIsBetter"
+          output-file-path: benchmark-results.json
+          external-data-json-path: ./benches/benchmark-data.json
+          # Fail the workflow if any benchmark regresses by more than 30% relative
+          # to the stored baseline. CI runners have variable load, so a threshold
+          # that is too tight will cause spurious failures; 130% is a reasonable
+          # starting point and can be tightened once baseline stability is known.
+          alert-threshold: "130%"
+          fail-on-alert: true
+          # GitHub API token to make a commit comment
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Post a comment on the commit when an alert is triggered, listing which
+          # benchmarks regressed and by how much.
+          comment-on-alert: true
+          # Always write a job summary table, not only when a regression is found.
+          summary-always: true
+
+      - name: Commit updated baseline
+        # Only update the stored baseline when code is merged to main. On pull
+        # requests the job reads the baseline for comparison but must not overwrite
+        # it, since the PR branch has not yet been reviewed and merged.
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "github-actions@github.com"
+          git add benches/benchmark-data.json
+          git diff --staged --quiet || git commit -m "chore: update benchmark baseline [skip ci]"
+          git push

benches/assets.rs

@@ -32,9 +32,11 @@ const YEAR: u32 = 2030;
 /// competing candidate technologies (petrol, diesel, electric and hybrid cars), making it a
 /// representative, non-trivial case for asset selection.
 const COMMODITY_ID: &str = "TPASKM";
-/// The range of numbers of competing candidate technologies to benchmark, to see how
-/// [`select_best_assets`] scales with the size of the search space.
-const N_TECHNOLOGIES_RANGE: std::ops::RangeInclusive<usize> = 1..=20;
+/// The numbers of competing candidate technologies to benchmark, stepped in fives, to see how
+/// [`select_best_assets`] scales with the size of the search space. Values are zero-padded to
+/// two digits when used as benchmark IDs so that names sort correctly on the filesystem
+/// (e.g. `05` before `10` rather than `10` before `5`).
+const N_TECHNOLOGIES: &[usize] = &[1, 5, 10, 15, 20];
 
 /// Extract the `two_outputs` example model to a temporary directory, load it, and create a
 /// (non-debug) [`DataWriter`] for it.
@@ -163,7 +165,7 @@ fn criterion_benchmark(c: &mut Criterion) {
     );
 
     // Real candidate technologies for this market, used as templates to build up to
-    // `N_TECHNOLOGIES_RANGE.end()` synthetic competing technologies
+    // `N_TECHNOLOGIES` synthetic competing technologies
     let templates: Vec<Arc<Process>> = agent
         .iter_search_space(region_id, &commodity.id, YEAR)
         .cloned()
@@ -184,7 +186,7 @@ fn criterion_benchmark(c: &mut Criterion) {
             .sample_size(20)
             .measurement_time(Duration::from_secs(3));
 
-        for n in N_TECHNOLOGIES_RANGE {
+        for &n in N_TECHNOLOGIES {
             // Give the agent a synthetic search space of `n` competing technologies
             let mut agent = agent.clone();
             agent.search_space.insert(
@@ -210,40 +212,44 @@ fn criterion_benchmark(c: &mut Criterion) {
                 commodity_portion,
             );
 
-            group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| {
-                b.iter_batched(
-                    || {
-                        (
-                            opt_assets.clone(),
-                            agent_addition_limits.clone(),
-                            demand.clone(),
-                        )
-                    },
-                    |(opt_assets, agent_addition_limits, demand)| {
-                        let run = || {
-                            select_best_assets(
-                                black_box(&model),
-                                opt_assets,
-                                agent_addition_limits,
-                                black_box(commodity),
-                                black_box(&agent),
-                                black_box(region_id),
-                                black_box(&prices),
-                                demand,
-                                black_box(YEAR),
-                                &mut writer,
+            group.bench_with_input(
+                BenchmarkId::from_parameter(format!("{n:02}")),
+                &n,
+                |b, _| {
+                    b.iter_batched(
+                        || {
+                            (
+                                opt_assets.clone(),
+                                agent_addition_limits.clone(),
+                                demand.clone(),
                             )
-                            .expect("select_best_assets failed")
-                        };
-                        if *use_parallel {
-                            run()
-                        } else {
-                            sequential_pool.install(run)
-                        }
-                    },
-                    BatchSize::SmallInput,
-                );
-            });
+                        },
+                        |(opt_assets, agent_addition_limits, demand)| {
+                            let run = || {
+                                select_best_assets(
+                                    black_box(&model),
+                                    opt_assets,
+                                    agent_addition_limits,
+                                    black_box(commodity),
+                                    black_box(&agent),
+                                    black_box(region_id),
+                                    black_box(&prices),
+                                    demand,
+                                    black_box(YEAR),
+                                    &mut writer,
+                                )
+                                .expect("select_best_assets failed")
+                            };
+                            if *use_parallel {
+                                run()
+                            } else {
+                                sequential_pool.install(run)
+                            }
+                        },
+                        BatchSize::SmallInput,
+                    );
+                },
+            );
         }
         group.finish();
     }

benches/convert_criterion_output.py

@@ -0,0 +1,39 @@
+"""Convert Criterion benchmark results to the JSON format expected by
+github-action-benchmark (customSmallerIsBetter).
+
+Criterion writes one estimates.json file per benchmark under:
+    target/criterion/<group>/<bench_id>/new/estimates.json
+
+where any '/' in a group name is sanitised to '_' by Criterion when creating
+directory names.  The benchmark name used for tracking is derived by stripping
+the leading target/criterion/ prefix and the trailing /new/estimates.json
+suffix, then joining the remaining path components with '/'.
+
+The output is written to benchmark-results.json in the repository root, in the
+format:
+    [{"name": "<name>", "value": <median_ns>, "unit": "ns"}, ...]
+
+All paths are resolved relative to the repository root (the parent directory of
+the directory containing this script), so the script can be invoked from any
+working directory.
+"""
+
+import json
+from pathlib import Path
+
+repo_root = Path(__file__).parent.parent
+criterion_dir = repo_root / "target" / "criterion"
+output_file = repo_root / "benchmark-results.json"
+
+results = []
+for estimates_file in sorted(criterion_dir.rglob("new/estimates.json")):
+    parts = estimates_file.relative_to(criterion_dir).parts[:-2]
+    name = "/".join(parts)
+    data = json.loads(estimates_file.read_text())
+    value = data["median"]["point_estimate"]
+    results.append({"name": name, "value": value, "unit": "ns"})
+
+output_file.write_text(json.dumps(results, indent=2))
+print(f"Converted {len(results)} Criterion benchmark results to {output_file}")
+for r in results:
+    print(f"  {r['name']}: {r['value']:.0f} ns")