diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 0099fe02d..f28420db3 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -40,3 +40,53 @@ jobs: uses: ./.github/actions/codecov with: token: ${{ secrets.CODECOV_TOKEN }} + + benchmark: + name: Run benchmarks + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write # needed to push the updated baseline back to the repo + pull-requests: write # needed to post a comment when an alert is triggered + steps: + - uses: actions/checkout@v7 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Run cargo bench + shell: bash + run: cargo bench --features bench --no-fail-fast + + - name: Convert Criterion results to JSON + run: python3 benches/convert_criterion_output.py + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "customSmallerIsBetter" + output-file-path: benchmark-results.json + external-data-json-path: ./benches/benchmark-data.json + # Fail the workflow if any benchmark regresses by more than 30% relative + # to the stored baseline. CI runners have variable load, so a threshold + # that is too tight will cause spurious failures; 130% is a reasonable + # starting point and can be tightened once baseline stability is known. + alert-threshold: "130%" + fail-on-alert: true + # GitHub API token to make a commit comment + github-token: ${{ secrets.GITHUB_TOKEN }} + # Post a comment on the commit when an alert is triggered, listing which + # benchmarks regressed and by how much. + comment-on-alert: true + # Always write a job summary table, not only when a regression is found. + summary-always: true + + - name: Commit updated baseline + # Only update the stored baseline when code is merged to main. On pull + # requests the job reads the baseline for comparison but must not overwrite + # it, since the PR branch has not yet been reviewed and merged. + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: | + git config user.name "GitHub Actions" + git config user.email "github-actions@github.com" + git add benches/benchmark-data.json + git diff --staged --quiet || git commit -m "chore: update benchmark baseline [skip ci]" + git push diff --git a/benches/assets.rs b/benches/assets.rs index 431e64054..bc7bb2f92 100644 --- a/benches/assets.rs +++ b/benches/assets.rs @@ -32,9 +32,11 @@ const YEAR: u32 = 2030; /// competing candidate technologies (petrol, diesel, electric and hybrid cars), making it a /// representative, non-trivial case for asset selection. const COMMODITY_ID: &str = "TPASKM"; -/// The range of numbers of competing candidate technologies to benchmark, to see how -/// [`select_best_assets`] scales with the size of the search space. -const N_TECHNOLOGIES_RANGE: std::ops::RangeInclusive = 1..=20; +/// The numbers of competing candidate technologies to benchmark, stepped in fives, to see how +/// [`select_best_assets`] scales with the size of the search space. Values are zero-padded to +/// two digits when used as benchmark IDs so that names sort correctly on the filesystem +/// (e.g. `05` before `10` rather than `10` before `5`). +const N_TECHNOLOGIES: &[usize] = &[1, 5, 10, 15, 20]; /// Extract the `two_outputs` example model to a temporary directory, load it, and create a /// (non-debug) [`DataWriter`] for it. @@ -163,7 +165,7 @@ fn criterion_benchmark(c: &mut Criterion) { ); // Real candidate technologies for this market, used as templates to build up to - // `N_TECHNOLOGIES_RANGE.end()` synthetic competing technologies + // `N_TECHNOLOGIES` synthetic competing technologies let templates: Vec> = agent .iter_search_space(region_id, &commodity.id, YEAR) .cloned() @@ -184,7 +186,7 @@ fn criterion_benchmark(c: &mut Criterion) { .sample_size(20) .measurement_time(Duration::from_secs(3)); - for n in N_TECHNOLOGIES_RANGE { + for &n in N_TECHNOLOGIES { // Give the agent a synthetic search space of `n` competing technologies let mut agent = agent.clone(); agent.search_space.insert( @@ -210,40 +212,44 @@ fn criterion_benchmark(c: &mut Criterion) { commodity_portion, ); - group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { - b.iter_batched( - || { - ( - opt_assets.clone(), - agent_addition_limits.clone(), - demand.clone(), - ) - }, - |(opt_assets, agent_addition_limits, demand)| { - let run = || { - select_best_assets( - black_box(&model), - opt_assets, - agent_addition_limits, - black_box(commodity), - black_box(&agent), - black_box(region_id), - black_box(&prices), - demand, - black_box(YEAR), - &mut writer, + group.bench_with_input( + BenchmarkId::from_parameter(format!("{n:02}")), + &n, + |b, _| { + b.iter_batched( + || { + ( + opt_assets.clone(), + agent_addition_limits.clone(), + demand.clone(), ) - .expect("select_best_assets failed") - }; - if *use_parallel { - run() - } else { - sequential_pool.install(run) - } - }, - BatchSize::SmallInput, - ); - }); + }, + |(opt_assets, agent_addition_limits, demand)| { + let run = || { + select_best_assets( + black_box(&model), + opt_assets, + agent_addition_limits, + black_box(commodity), + black_box(&agent), + black_box(region_id), + black_box(&prices), + demand, + black_box(YEAR), + &mut writer, + ) + .expect("select_best_assets failed") + }; + if *use_parallel { + run() + } else { + sequential_pool.install(run) + } + }, + BatchSize::SmallInput, + ); + }, + ); } group.finish(); } diff --git a/benches/convert_criterion_output.py b/benches/convert_criterion_output.py new file mode 100644 index 000000000..88f5968b7 --- /dev/null +++ b/benches/convert_criterion_output.py @@ -0,0 +1,39 @@ +"""Convert Criterion benchmark results to the JSON format expected by +github-action-benchmark (customSmallerIsBetter). + +Criterion writes one estimates.json file per benchmark under: + target/criterion///new/estimates.json + +where any '/' in a group name is sanitised to '_' by Criterion when creating +directory names. The benchmark name used for tracking is derived by stripping +the leading target/criterion/ prefix and the trailing /new/estimates.json +suffix, then joining the remaining path components with '/'. + +The output is written to benchmark-results.json in the repository root, in the +format: + [{"name": "", "value": , "unit": "ns"}, ...] + +All paths are resolved relative to the repository root (the parent directory of +the directory containing this script), so the script can be invoked from any +working directory. +""" + +import json +from pathlib import Path + +repo_root = Path(__file__).parent.parent +criterion_dir = repo_root / "target" / "criterion" +output_file = repo_root / "benchmark-results.json" + +results = [] +for estimates_file in sorted(criterion_dir.rglob("new/estimates.json")): + parts = estimates_file.relative_to(criterion_dir).parts[:-2] + name = "/".join(parts) + data = json.loads(estimates_file.read_text()) + value = data["median"]["point_estimate"] + results.append({"name": name, "value": value, "unit": "ns"}) + +output_file.write_text(json.dumps(results, indent=2)) +print(f"Converted {len(results)} Criterion benchmark results to {output_file}") +for r in results: + print(f" {r['name']}: {r['value']:.0f} ns")