From 00ec3adc2dee05b053c56392da0864ae9ba143ec Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Fri, 21 Aug 2026 07:50:23 +0000 Subject: [PATCH 1/2] feat(mri): expose skip_initial_run on leaderboard creation Threads the backend's skipInitialRun flag through create_leaderboard, defaulting to False so existing callers keep queueing the initial run. Co-Authored-By: Claude Opus 5 Co-Authored-By: lino <68745352+LinoGiger@users.noreply.github.com> --- docs/mri_advanced.md | 19 +++++++ .../benchmark/rapidata_benchmark.py | 6 +- .../test_leaderboard_skip_initial_run.py | 55 +++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/rapidata_client/benchmark/test_leaderboard_skip_initial_run.py diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index 6c9b1583d2..37938443db 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -134,6 +134,25 @@ leaderboard = benchmark.create_leaderboard( ) ``` +### Skipping the Initial Run + +Creating a leaderboard normally starts an initial run that evaluates the models +already in the benchmark against each other. Pass `skip_initial_run=True` to +create the leaderboard without it: + +```python +leaderboard = benchmark.create_leaderboard( + name="Staged Evaluation", + instruction="Which image do you prefer?", + skip_initial_run=True, +) +``` + +The leaderboard starts with no responses and therefore no standings. Everything +else still works: a model added afterwards is compared against the whole existing +field, and the leaderboard can be boosted from the app. Use this when you want to +decide what gets evaluated first rather than paying for a full round up front. + ### Vote Aggregation Each matchup — one comparison of two models on one prompt — collects several diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 2009a68747..6bedce34a3 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -630,6 +630,7 @@ def create_leaderboard( included_tags: list[str] | None = None, excluded_tags: list[str] | None = None, vote_aggregation: VoteAggregation = VoteAggregation.MAJORITY_VOTE, + skip_initial_run: bool = False, ) -> RapidataLeaderboard: """ Creates a new leaderboard for the benchmark. @@ -647,6 +648,7 @@ def create_leaderboard( included_tags: Restricts **which of the benchmark's prompts this leaderboard collects matchups for**: only prompts carrying at least one of these tag values are used. When empty or not specified (the default) every prompt is eligible. Note that a non-empty list drops untagged prompts. (default: None) excluded_tags: Prompt tag values to skip when collecting matchups. Always wins over ``included_tags`` — a prompt carrying both an included and an excluded tag is skipped. (default: None) vote_aggregation: How the responses on a single matchup are aggregated into that matchup's result. :attr:`VoteAggregation.MAJORITY_VOTE` (the default) collapses each matchup to one win for the majority side, ties split 0.5/0.5, so every matchup weighs the same no matter how many responses it collected. :attr:`VoteAggregation.ALL_VOTES` counts every individual response as its own matchup, which lets heavily-answered matchups dominate the standings. Changeable afterwards via :attr:`RapidataLeaderboard.vote_aggregation`. + skip_initial_run: Whether to skip the initial run that evaluates the models already in the benchmark against each other. Adding a model afterwards still compares it against the whole existing field, and boosting the leaderboard still works — you just start with no responses collected and therefore no standings. Set this when you want to choose what gets evaluated first instead of paying for a full round. (default: False) Do not confuse either tag argument with the ``tags`` argument of :meth:`RapidataLeaderboard.get_standings`: that filters the standings you read @@ -693,7 +695,7 @@ def create_leaderboard( ) logger.info( - "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, audience_id %s, settings %s, included_tags %s, excluded_tags %s, vote_aggregation %s", + "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, audience_id %s, settings %s, included_tags %s, excluded_tags %s, vote_aggregation %s, skip_initial_run %s", name, instruction, show_prompt, @@ -706,6 +708,7 @@ def create_leaderboard( included_tags, excluded_tags, vote_aggregation.name, + skip_initial_run, ) leaderboard_result = ( @@ -723,6 +726,7 @@ def create_leaderboard( includedTags=included_tags, excludedTags=excluded_tags, voteAggregation=vote_aggregation._to_backend_model(), + skipInitialRun=skip_initial_run, featureFlags=( [setting._to_feature_flag() for setting in settings] if settings diff --git a/tests/rapidata_client/benchmark/test_leaderboard_skip_initial_run.py b/tests/rapidata_client/benchmark/test_leaderboard_skip_initial_run.py new file mode 100644 index 0000000000..5890f4671c --- /dev/null +++ b/tests/rapidata_client/benchmark/test_leaderboard_skip_initial_run.py @@ -0,0 +1,55 @@ +"""Tests for ``skip_initial_run`` on leaderboard creation. + +The flag suppresses the initial run that evaluates the benchmark's existing +models against each other. It is create-only: the backend decides it at creation +time and does not record it on the leaderboard, so there is nothing to read back. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from rapidata.api_client.models.vote_aggregation import ( + VoteAggregation as VoteAggregationModel, +) +from rapidata.rapidata_client.benchmark.rapidata_benchmark import RapidataBenchmark + + +def _make_benchmark() -> RapidataBenchmark: + svc = MagicMock() + svc.environment = "rapidata.ai" + create = svc.leaderboard.leaderboard_api.leaderboard_post + create.return_value.benchmark_id = "bm-1" + create.return_value.id = "lb-1" + create.return_value.response_budget = 2000 + create.return_value.min_responses = 3 + create.return_value.vote_aggregation = VoteAggregationModel.MAJORITYVOTE + return RapidataBenchmark("bm", "bm-1", svc) + + +def _sent_payload(benchmark: RapidataBenchmark): + create = benchmark._openapi_service.leaderboard.leaderboard_api.leaderboard_post + return create.call_args.kwargs["create_leaderboard_endpoint_input"] + + +def test_create_leaderboard_threads_skip_initial_run_to_the_wire() -> None: + benchmark = _make_benchmark() + benchmark.create_leaderboard( + name="lb", + instruction="Which is better?", + skip_initial_run=True, + ) + + assert _sent_payload(benchmark).skip_initial_run is True + + +def test_create_leaderboard_runs_initially_by_default() -> None: + benchmark = _make_benchmark() + benchmark.create_leaderboard( + name="lb", + instruction="Which is better?", + ) + + # Explicitly False rather than omitted, so the request never relies on the + # server-side default to keep today's behaviour. + assert _sent_payload(benchmark).skip_initial_run is False From 6991e0792770e0d280c6d0f8ccaf35d862bddeb1 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Fri, 21 Aug 2026 07:53:34 +0000 Subject: [PATCH 2/2] docs(mri): drop the skip_initial_run section from the MRI guide Keeping it to the create_leaderboard docstring: the flag is there for people who already know they want it, not a pattern to point users at. Co-Authored-By: Claude Opus 5 Co-Authored-By: lino <68745352+LinoGiger@users.noreply.github.com> --- docs/mri_advanced.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index 37938443db..6c9b1583d2 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -134,25 +134,6 @@ leaderboard = benchmark.create_leaderboard( ) ``` -### Skipping the Initial Run - -Creating a leaderboard normally starts an initial run that evaluates the models -already in the benchmark against each other. Pass `skip_initial_run=True` to -create the leaderboard without it: - -```python -leaderboard = benchmark.create_leaderboard( - name="Staged Evaluation", - instruction="Which image do you prefer?", - skip_initial_run=True, -) -``` - -The leaderboard starts with no responses and therefore no standings. Everything -else still works: a model added afterwards is compared against the whole existing -field, and the leaderboard can be boosted from the app. Use this when you want to -decide what gets evaluated first rather than paying for a full round up front. - ### Vote Aggregation Each matchup — one comparison of two models on one prompt — collects several