diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 2009a68747..6bedce34a3 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -630,6 +630,7 @@ def create_leaderboard( included_tags: list[str] | None = None, excluded_tags: list[str] | None = None, vote_aggregation: VoteAggregation = VoteAggregation.MAJORITY_VOTE, + skip_initial_run: bool = False, ) -> RapidataLeaderboard: """ Creates a new leaderboard for the benchmark. @@ -647,6 +648,7 @@ def create_leaderboard( included_tags: Restricts **which of the benchmark's prompts this leaderboard collects matchups for**: only prompts carrying at least one of these tag values are used. When empty or not specified (the default) every prompt is eligible. Note that a non-empty list drops untagged prompts. (default: None) excluded_tags: Prompt tag values to skip when collecting matchups. Always wins over ``included_tags`` — a prompt carrying both an included and an excluded tag is skipped. (default: None) vote_aggregation: How the responses on a single matchup are aggregated into that matchup's result. :attr:`VoteAggregation.MAJORITY_VOTE` (the default) collapses each matchup to one win for the majority side, ties split 0.5/0.5, so every matchup weighs the same no matter how many responses it collected. :attr:`VoteAggregation.ALL_VOTES` counts every individual response as its own matchup, which lets heavily-answered matchups dominate the standings. Changeable afterwards via :attr:`RapidataLeaderboard.vote_aggregation`. + skip_initial_run: Whether to skip the initial run that evaluates the models already in the benchmark against each other. Adding a model afterwards still compares it against the whole existing field, and boosting the leaderboard still works — you just start with no responses collected and therefore no standings. Set this when you want to choose what gets evaluated first instead of paying for a full round. (default: False) Do not confuse either tag argument with the ``tags`` argument of :meth:`RapidataLeaderboard.get_standings`: that filters the standings you read @@ -693,7 +695,7 @@ def create_leaderboard( ) logger.info( - "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, audience_id %s, settings %s, included_tags %s, excluded_tags %s, vote_aggregation %s", + "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, audience_id %s, settings %s, included_tags %s, excluded_tags %s, vote_aggregation %s, skip_initial_run %s", name, instruction, show_prompt, @@ -706,6 +708,7 @@ def create_leaderboard( included_tags, excluded_tags, vote_aggregation.name, + skip_initial_run, ) leaderboard_result = ( @@ -723,6 +726,7 @@ def create_leaderboard( includedTags=included_tags, excludedTags=excluded_tags, voteAggregation=vote_aggregation._to_backend_model(), + skipInitialRun=skip_initial_run, featureFlags=( [setting._to_feature_flag() for setting in settings] if settings diff --git a/tests/rapidata_client/benchmark/test_leaderboard_skip_initial_run.py b/tests/rapidata_client/benchmark/test_leaderboard_skip_initial_run.py new file mode 100644 index 0000000000..5890f4671c --- /dev/null +++ b/tests/rapidata_client/benchmark/test_leaderboard_skip_initial_run.py @@ -0,0 +1,55 @@ +"""Tests for ``skip_initial_run`` on leaderboard creation. + +The flag suppresses the initial run that evaluates the benchmark's existing +models against each other. It is create-only: the backend decides it at creation +time and does not record it on the leaderboard, so there is nothing to read back. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from rapidata.api_client.models.vote_aggregation import ( + VoteAggregation as VoteAggregationModel, +) +from rapidata.rapidata_client.benchmark.rapidata_benchmark import RapidataBenchmark + + +def _make_benchmark() -> RapidataBenchmark: + svc = MagicMock() + svc.environment = "rapidata.ai" + create = svc.leaderboard.leaderboard_api.leaderboard_post + create.return_value.benchmark_id = "bm-1" + create.return_value.id = "lb-1" + create.return_value.response_budget = 2000 + create.return_value.min_responses = 3 + create.return_value.vote_aggregation = VoteAggregationModel.MAJORITYVOTE + return RapidataBenchmark("bm", "bm-1", svc) + + +def _sent_payload(benchmark: RapidataBenchmark): + create = benchmark._openapi_service.leaderboard.leaderboard_api.leaderboard_post + return create.call_args.kwargs["create_leaderboard_endpoint_input"] + + +def test_create_leaderboard_threads_skip_initial_run_to_the_wire() -> None: + benchmark = _make_benchmark() + benchmark.create_leaderboard( + name="lb", + instruction="Which is better?", + skip_initial_run=True, + ) + + assert _sent_payload(benchmark).skip_initial_run is True + + +def test_create_leaderboard_runs_initially_by_default() -> None: + benchmark = _make_benchmark() + benchmark.create_leaderboard( + name="lb", + instruction="Which is better?", + ) + + # Explicitly False rather than omitted, so the request never relies on the + # server-side default to keep today's behaviour. + assert _sent_payload(benchmark).skip_initial_run is False