diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 10411ad069..5bc1a43ccd 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -6,6 +6,10 @@ on: runner: required: true type: string + node-count: + description: "Total Slurm nodes required by this benchmark" + required: true + type: string priority: description: "Higher-is-sooner CI priority score" required: true @@ -254,22 +258,51 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && ( - inputs.skip-queue-pr != '' && - format( - '["self-hosted",{0},{1},{2},{3}]', - toJSON(inputs.runner), - toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), - toJSON(format('ci-attempt-{0}', github.run_attempt)), - toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' && + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted",{0},{1},{2},{3},{4}]', + toJSON(inputs.runner), + toJSON(format('nodes:{0}', inputs.node-count)), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + ) || + format( + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON(format('nodes:{0}', inputs.node-count)), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) ) || - format( - '["self-hosted",{0},{1},{2}]', - toJSON(inputs.runner), - toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), - toJSON(format('ci-attempt-{0}', github.run_attempt)) + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + ) || + format( + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) ) ) || - format('[{0}]', toJSON(inputs.runner)) + ( + vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' && + format( + '[{0},{1}]', + toJSON(inputs.runner), + toJSON(format('nodes:{0}', inputs.node-count)) + ) || + format('[{0}]', toJSON(inputs.runner)) + ) ) }} timeout-minutes: 480 name: >- diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 46a80c8743..16fad85a5f 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -202,22 +202,47 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && ( - inputs.skip-queue-pr != '' && - format( - '["self-hosted",{0},{1},{2},{3}]', - toJSON(inputs.runner), - toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), - toJSON(format('ci-attempt-{0}', github.run_attempt)), - toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' && + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted",{0},{1},{2},{3},{4}]', + toJSON(inputs.runner), + toJSON('nodes:1'), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + ) || + format( + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON('nodes:1'), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) ) || - format( - '["self-hosted",{0},{1},{2}]', - toJSON(inputs.runner), - toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), - toJSON(format('ci-attempt-{0}', github.run_attempt)) + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + ) || + format( + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) ) ) || - format('[{0}]', toJSON(inputs.runner)) + ( + vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' && + format('[{0},{1}]', toJSON(inputs.runner), toJSON('nodes:1')) || + format('[{0}]', toJSON(inputs.runner)) + ) ) }} timeout-minutes: 500 name: >- diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 34fba860f4..297a24510e 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -152,6 +152,7 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + node-count: ${{ matrix.config.node-count }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} @@ -203,6 +204,7 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + node-count: ${{ matrix.config.node-count }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} @@ -342,6 +344,7 @@ jobs: osl: '0' max-model-len: '0' runner: ${{ matrix.config.runner }} + node-count: ${{ matrix.config.node-count }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 9d52fda0b9..dc40de81fb 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -457,6 +457,7 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + node-count: ${{ matrix.config.node-count }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} @@ -651,6 +652,7 @@ jobs: osl: '0' max-model-len: '0' runner: ${{ matrix.config.runner }} + node-count: ${{ matrix.config.node-count }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} @@ -817,6 +819,7 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + node-count: ${{ matrix.config.node-count }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index 34565e2810..83a21a8117 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -6,6 +6,9 @@ version: 1 base-score: 1.0 adjustments: + # Preserve business priority as the primary key, then prefer smaller Slurm + # allocations within an otherwise identical class. + additional-node: -0.001 event: push: 2.0 multi-node: 1.25 diff --git a/utils/ci_priority.py b/utils/ci_priority.py index 89b51d3ed5..540d36e759 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -113,6 +113,7 @@ def _entry_from_criteria( ) else "" ), + "node-count": entry.get("node-count", 1), } @@ -141,6 +142,15 @@ def calculate_priority( score = _decimal(policy["base-score"]) score += _decimal(adjustments.get("event", {}).get(context.event_name, 0)) + node_count = entry.get("node-count", 1) + if ( + not isinstance(node_count, int) + or isinstance(node_count, bool) + or node_count < 1 + ): + raise ValueError(f"node-count must be a positive integer, got {node_count!r}") + score += _decimal(adjustments.get("additional-node", 0)) * (node_count - 1) + if entry.get("prefill") is not None: score += _decimal(adjustments.get("multi-node", 0)) if entry.get("scenario-type") == "agentic-coding": diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 7df1a75b64..a21757fe65 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -1,20 +1,24 @@ +import argparse import fnmatch import json -import argparse +import math +import re import sys from decimal import Decimal from pathlib import Path +import yaml + # Ensure sibling modules are importable regardless of how script is invoked sys.path.insert(0, str(Path(__file__).resolve().parent)) from validation import ( - validate_matrix_entry, - validate_agentic_matrix_entry, + DEFAULT_AGENTIC_DURATION_SECONDS, + Fields, load_config_files, load_runner_file, - Fields, - DEFAULT_AGENTIC_DURATION_SECONDS, + validate_agentic_matrix_entry, + validate_matrix_entry, ) seq_len_stoi = { @@ -82,6 +86,140 @@ def runner_gpus_per_node(runner: str, runner_data: dict) -> int: return runner_hardware_int(runner, runner_data, Fields.GPUS_PER_NODE.value) +def _hardware_family(label: str) -> str: + """Return the GPU family encoded in a runner or cluster label.""" + return label.removeprefix("cluster:").split("-", 1)[0] + + +def scheduling_gpus_per_node(label: str, runner_data: dict) -> int: + """Resolve GPUs per node for an abstract runner or worker hardware label. + + Scheduling labels such as ``b200-multinode`` do not currently duplicate + the hardware facts stored under ``cluster:b200-dgxc``. Fall back to the + GPU family when the exact label has no hardware record, while rejecting + ambiguous families that disagree about node shape. + """ + hardware = runner_hardware(runner_data) + exact = hardware.get(label) + if exact is not None: + return exact[Fields.GPUS_PER_NODE.value] + + family = _hardware_family(label) + matches = { + facts[Fields.GPUS_PER_NODE.value] + for hardware_label, facts in hardware.items() + if _hardware_family(hardware_label) == family + } + if len(matches) == 1: + return matches.pop() + if not matches: + raise ValueError( + f"Cannot resolve {Fields.GPUS_PER_NODE.value} for '{label}'" + ) + raise ValueError( + f"Ambiguous {Fields.GPUS_PER_NODE.value} for '{label}': {sorted(matches)}" + ) + + +def _worker_node_override(worker: dict, setting_name: str) -> int | None: + """Read an explicit role node count from additional settings.""" + pattern = re.compile(rf"^{re.escape(setting_name)}=(\d+)$") + values = [] + for setting in worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or []: + match = pattern.match(setting) + if match: + values.append(int(match.group(1))) + if not values: + return None + if len(set(values)) != 1 or values[0] <= 0: + raise ValueError(f"Conflicting or invalid {setting_name} settings: {values}") + return values[0] + + +def recipe_node_count(prefill: dict, decode: dict) -> int | None: + """Read the authoritative node count from a checked-in srt-slurm recipe.""" + config_files = { + setting.split("=", 1)[1] + for worker in (prefill, decode) + for setting in (worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or []) + if setting.startswith("CONFIG_FILE=") + } + if not config_files: + return None + if len(config_files) != 1: + raise ValueError(f"Conflicting CONFIG_FILE settings: {sorted(config_files)}") + + relative_path = config_files.pop().removeprefix("recipes/") + recipe_path = ( + Path(__file__).resolve().parents[2] + / "benchmarks" + / "multi_node" + / "srt-slurm-recipes" + / relative_path + ) + if not recipe_path.exists(): + # Some srt-slurm recipes live only in the runtime image. Their master + # config topology remains the best available scheduling estimate. + return None + + resources = yaml.safe_load(recipe_path.read_text())["resources"] + if "agg_nodes" in resources: + return int(resources["agg_nodes"]) + if "prefill_nodes" in resources and "decode_nodes" in resources: + return int(resources["prefill_nodes"]) + int(resources["decode_nodes"]) + raise ValueError(f"Recipe has no supported node resource fields: {recipe_path}") + + +def worker_node_count( + worker: dict, + role: str, + runner: str, + runner_data: dict, +) -> int: + """Return physical nodes consumed by one prefill or decode role.""" + override = _worker_node_override(worker, f"{role.upper()}_NODES") + if override is not None: + return override + + hardware_label = worker.get(Fields.HARDWARE.value) or runner + gpus_per_node = scheduling_gpus_per_node(hardware_label, runner_data) + total_gpus = ( + worker[Fields.NUM_WORKER.value] + * worker[Fields.TP.value] + * worker.get(Fields.PP.value, 1) + * worker.get(Fields.PCP_SIZE.value, 1) + ) + return math.ceil(total_gpus / gpus_per_node) + + +def multinode_node_count( + prefill: dict, + decode: dict, + runner: str, + runner_data: dict, +) -> int: + """Return the total Slurm node request represented by a matrix row.""" + recipe_count = recipe_node_count(prefill, decode) + if recipe_count is not None: + return recipe_count + return ( + worker_node_count(prefill, "prefill", runner, runner_data) + + worker_node_count(decode, "decode", runner, runner_data) + ) + + +def add_multinode_node_count(entry: dict, runner_data: dict) -> dict: + """Annotate a generated multi-node row when runner metadata is available.""" + if runner_data: + entry[Fields.NODE_COUNT.value] = multinode_node_count( + entry[Fields.PREFILL.value], + entry[Fields.DECODE.value], + entry[Fields.RUNNER.value], + runner_data, + ) + return entry + + def effective_gpu_count(benchmark: dict) -> int: """Return GPUs used by a single-node TP/PP/PCP topology.""" return ( @@ -568,6 +706,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.RUN_EVAL.value: False, # Default, may be overridden by mark_eval_entries } entry.update(component_metadata(bmk, val)) + add_multinode_node_count(entry, runner_data) validate_matrix_entry(entry, is_multinode) matrix_values.append(entry) @@ -787,6 +926,7 @@ def generate_full_sweep(args, all_config_data, runner_data): if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend entry.update(component_metadata(bmk, val)) + add_multinode_node_count(entry, runner_data) validate_agentic_matrix_entry(entry) matrix_values.append(entry) else: @@ -939,6 +1079,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.RUN_EVAL.value: False, } entry.update(component_metadata(bmk, val)) + add_multinode_node_count(entry, runner_data) matrix_values.append(validate_matrix_entry(entry, is_multinode=True)) else: # Single-node config @@ -1085,6 +1226,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend entry.update(component_metadata(bmk, val)) + add_multinode_node_count(entry, runner_data) matrix_values.append(validate_agentic_matrix_entry(entry)) else: for conc in conc_values: diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 1ad4e3c922..6be6f74cf1 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -1,20 +1,78 @@ """Comprehensive tests for generate_sweep_configs.py""" -import pytest + import argparse import copy + +import pytest + from generate_sweep_configs import ( MIN_EVAL_CONC, - seq_len_stoi, - seq_len_itos, - seq_len_to_str, + apply_node_type_defaults, + expand_config_keys, generate_full_sweep, generate_test_config_sweep, - mark_eval_entries, mark_all_eval_entries, - apply_node_type_defaults, - expand_config_keys, + mark_eval_entries, + multinode_node_count, + seq_len_itos, + seq_len_stoi, + seq_len_to_str, ) +def test_multinode_node_count_uses_role_gpu_footprints(sample_runner_config): + prefill = {"num-worker": 3, "tp": 2, "pp": 1, "pcp-size": 1} + decode = {"num-worker": 2, "tp": 8, "pp": 1, "pcp-size": 1} + + assert multinode_node_count( + prefill, decode, "cluster:b300-nv", sample_runner_config + ) == 3 + + +def test_multinode_node_count_honors_explicit_role_node_settings(): + prefill = { + "num-worker": 1, + "tp": 8, + "additional-settings": ["PREFILL_NODES=2"], + } + decode = { + "num-worker": 1, + "tp": 8, + "additional-settings": ["DECODE_NODES=1"], + } + + assert multinode_node_count(prefill, decode, "unknown", {}) == 3 + + +def test_multinode_node_count_resolves_heterogeneous_worker_hardware( + sample_runner_config, +): + prefill = {"hardware": "gb200", "num-worker": 5, "tp": 4} + decode = {"hardware": "h100", "num-worker": 1, "tp": 8} + + assert multinode_node_count( + prefill, decode, "gb200", sample_runner_config + ) == 6 + + +def test_multinode_node_count_prefers_checked_in_recipe_resources( + sample_runner_config, +): + prefill = { + "num-worker": 3, + "tp": 1, + "additional-settings": [ + ( + "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/" + "disagg-gb200-3p1d-dep4-dep16.yaml" + ) + ], + } + decode = {"num-worker": 1, "tp": 1} + + assert multinode_node_count( + prefill, decode, "cluster:gb200-nv", sample_runner_config + ) == 7 + # ============================================================================= # Test Fixtures diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 9f19572f08..89cadba3dc 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -72,6 +72,7 @@ class Fields(Enum): AVAILABLE_CPU_DRAM_MIB = 'available-cpu-dram-mib' DRAM_UTILIZATION = 'dram-utilization' GPUS_PER_NODE = 'gpus-per-node' + NODE_COUNT = 'node-count' DURATION = 'duration' # Matrix entry fields @@ -227,6 +228,9 @@ class MultiNodeMatrixEntry(BaseModel): alias=Fields.SPEC_DECODING.value ) runner: str + node_count: Optional[int] = Field( + default=None, alias=Fields.NODE_COUNT.value, gt=0, strict=True + ) isl: int osl: int prefill: WorkerConfig @@ -318,6 +322,9 @@ class MultiNodeAgenticMatrixEntry(BaseModel): alias=Fields.SPEC_DECODING.value ) runner: str + node_count: Optional[int] = Field( + default=None, alias=Fields.NODE_COUNT.value, gt=0, strict=True + ) prefill: WorkerConfig decode: WorkerConfig conc: list[int] diff --git a/utils/runner_setup/RUNNER_SETUP.md b/utils/runner_setup/RUNNER_SETUP.md index 5ac57e0e6c..c170a661a6 100644 --- a/utils/runner_setup/RUNNER_SETUP.md +++ b/utils/runner_setup/RUNNER_SETUP.md @@ -180,6 +180,43 @@ self-hosted, Linux, X64, slurm, b200, b200-dsv4, cluster:b200-dgxc, b200-dgxc_00 Labels can be edited later on the runners settings page without re-registering. +### Static Slurm node-slot labels + +The optional node-slot scheduler limits how many GitHub jobs of one Slurm +allocation size can enter a fleet at once. A job that needs three nodes adds +`nodes:3` to `runs-on`. In a 12-runner pool, exactly `floor(12 / 3) = 4` +runners receive that label; `nodes:2` is present on six runners and `nodes:1` +is present on all twelve. + +Generate the label plan from the runner order already recorded in +`configs/runners.yaml`: + +```bash +python utils/runner_setup/plan_node_labels.py cluster:h200-dgxc +``` + +The plan assigns each `nodes:N` label to the first `floor(capacity / N)` +runners in the configured list. The sets are deliberately nested toward the +earliest runner names, matching the scheduler's deterministic runner order. +Apply the emitted labels through the runner settings page or GitHub API, then +set the repository variable `NODE_SLOT_SCHEDULER_ENABLED=true`. Until that +variable is enabled, workflow `runs-on` expressions omit the node-slot label. +The priority score also subtracts `0.001` per additional node, so otherwise +equal jobs are ordered as one-node, two-node, three-node, and so on without +overriding the existing business-priority signals. + +Multi-node matrix generation derives `node-count` from, in precedence order: + +1. checked-in srt-slurm recipe `resources`; +2. explicit `PREFILL_NODES` and `DECODE_NODES` settings; or +3. worker GPU footprints divided by the relevant hardware's GPUs per node. + +This is an admission-slot approximation, not weighted resource accounting. +It caps a uniform workload correctly (for example, four simultaneous +three-node jobs on 12 nodes), but a mixture of one-, two-, and three-node jobs +can still request more than 12 nodes in aggregate. Slurm remains the final +capacity authority and queues allocations that do not fit. + ## Storage layout The login node (where the runners live) and the Slurm compute nodes (where benchmarks diff --git a/utils/runner_setup/plan_node_labels.py b/utils/runner_setup/plan_node_labels.py new file mode 100755 index 0000000000..225011b513 --- /dev/null +++ b/utils/runner_setup/plan_node_labels.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Plan static GitHub runner labels for weighted Slurm admission slots.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import yaml + + +def plan_node_labels(runners: list[str]) -> dict[str, list[str]]: + """Assign ``nodes:N`` to the first ``floor(capacity / N)`` runners. + + The source order is intentional: ``configs/runners.yaml`` already records + the fleet's preferred runner order. Prefix assignment keeps the labels + nested, so larger allocations are available on progressively earlier + runner slots while ``nodes:1`` remains available everywhere. + """ + if not runners: + raise ValueError("runner pool cannot be empty") + if len(runners) != len(set(runners)): + raise ValueError("runner pool cannot contain duplicate names") + + capacity = len(runners) + return { + runner: [ + f"nodes:{node_count}" + for node_count in range(1, capacity + 1) + if index < capacity // node_count + ] + for index, runner in enumerate(runners) + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("pool", help="Runner label to treat as one Slurm pool") + parser.add_argument( + "--runner-config", + type=Path, + default=Path("configs/runners.yaml"), + ) + args = parser.parse_args() + + runner_config = yaml.safe_load(args.runner_config.read_text()) + try: + runners = runner_config["labels"][args.pool] + except KeyError as error: + raise SystemExit(f"Unknown runner pool: {args.pool}") from error + + print(json.dumps(plan_node_labels(runners), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 8bb7b11198..8898c4f937 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -54,6 +54,33 @@ def test_main_branch_jobs_receive_an_automatic_boost(): ) == Decimal("3.000") +def test_smaller_node_allocations_win_otherwise_equal_priority_ties(): + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority({**entry, "node-count": 1}, POLICY) == Decimal("1.000") + assert calculate_priority({**entry, "node-count": 2}, POLICY) == Decimal("0.999") + assert calculate_priority({**entry, "node-count": 3}, POLICY) == Decimal("0.998") + + +def test_node_count_tiebreaker_survives_classifier_projection(): + entry = {"runner": "h100", "framework": "trt", "node-count": 3} + + assert calculate_priority( + entry, + POLICY, + PriorityContext(criteria=frozenset()), + ) == Decimal("0.998") + + +@pytest.mark.parametrize("node_count", [0, -1, 1.5, True]) +def test_node_count_must_be_a_positive_integer(node_count): + with pytest.raises(ValueError, match="positive integer"): + calculate_priority( + {"runner": "h100", "framework": "trt", "node-count": node_count}, + POLICY, + ) + + def test_patchwork_score_uses_half_up_rounding(): policy = deepcopy(POLICY) policy["labels"]["patchwork"]["score"] = 0.7225