From 3a83b540ef5842c05255f42fc32edee86c22894b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 28 Jul 2026 09:09:33 -0500 Subject: [PATCH 1/2] refactor(runners): resolve model paths from runner config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the B200 DGXC model-path and srt-slurm alias table into configs/runners.yaml, add typed validation, and resolve it through a reusable helper. Preserve ordered fallbacks and the existing DSv4 operator override. 中文：将 B200 DGXC 的模型路径与 srt-slurm 别名映射迁移到 configs/runners.yaml，补充类型校验，并通过可复用工具统一解析。保留有序回退机制及现有的 DSv4 运维覆盖行为。 --- configs/runners.yaml | 80 ++++++++++++++ runners/launch_b200-dgxc.sh | 76 ++----------- .../matrix_logic/test_resolve_runner_model.py | 94 ++++++++++++++++ utils/matrix_logic/test_validation.py | 78 ++++++++++++++ utils/matrix_logic/validation.py | 44 ++++++++ utils/resolve_runner_model.py | 101 ++++++++++++++++++ 6 files changed, 403 insertions(+), 70 deletions(-) create mode 100644 utils/matrix_logic/test_resolve_runner_model.py create mode 100644 utils/resolve_runner_model.py diff --git a/configs/runners.yaml b/configs/runners.yaml index 851b821ba2..4235e429bd 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -330,3 +330,83 @@ hardware: cluster:mi355x-amds: available-cpu-dram-mib: 3_095_781 gpus-per-node: 8 +models: + # Cluster-local checkpoints used by launchers instead of downloading the + # portable HuggingFace model IDs in the benchmark configs. When multiple + # paths are listed, the first existing directory wins and the first entry is + # the deterministic fallback. + cluster:b200-dgxc: + dsr1: + fp4: + # Per-node /raid is not populated reliably across the cluster. + model-paths: + - /scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /lustre/fsw/models/dsr1-0528-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dsv4: + fp4: + model-paths: + - /lustre/fsw/models/deepseek-v4-pro + - /lustre/fsw/models/dsv4-pro + - /lustre/fsw/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-v4-pro + # Preserve the operator override supported by the previous launcher. + allow-model-path-override: true + qwen3.5: + bf16: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B + srt-slurm-model-prefix: qwen3.5 + fp8: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B-FP8 + srt-slurm-model-prefix: qwen3.5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4 + srt-slurm-model-prefix: qwen3.5-fp4 + glm5: + fp8: + model-paths: + - /lustre/fsw/models/GLM-5-FP8 + srt-slurm-model-prefix: glm5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/GLM-5-NVFP4 + srt-slurm-model-prefix: glm5-fp4 + kimik2.5: + int4: + model-paths: + - /lustre/fsw/models/Kimi-K2.5 + srt-slurm-model-prefix: kimik2.5 + fp4: + model-paths: + - /lustre/fsw/models/Kimi-K2.5-NVFP4 + srt-slurm-model-prefix: kimik2.5-fp4 + minimaxm2.5: + fp8: + model-paths: + - /lustre/fsw/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + gptoss: + fp4: + model-paths: + - /lustre/fsw/models/gpt-oss-120b + srt-slurm-model-prefix: gptoss + minimaxm3: + fp8: + # Day-zero model staged in the sa-shared-writable gharunners tree. + model-paths: + - /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: minimax-m3-mxfp8 + fp4: + model-paths: + - /scratch/fsw/models/MiniMax-M3-NVFP4 + srt-slurm-model-prefix: minimax-m3-nvfp4 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a276644575..d104b8e93e 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -6,78 +6,14 @@ SLURM_ACCOUNT="benchmark" set -x -# MODEL_PATH: Override with pre-downloaded paths on the shared Lustre tree. -# Bench scripts and srt-slurm yaml configs specify HuggingFace model IDs for -# portability, but we resolve to /lustre/fsw/models/* here to avoid repeated -# downloading on every dgxc node. Runs for both single-node and multinode -# launches. -# NOTE: per-node /raid/models/* would be faster but is only populated on a -# subset of dgxc nodes today, so we use Lustre for reliability. -if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2" - export SRT_SLURM_MODEL_PREFIX="dsr1" -elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/dsr1-0528-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - SELECTED_MODEL_PATH="" - if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then - SELECTED_MODEL_PATH="$MODEL_PATH" - else - for candidate in /lustre/fsw/models/deepseek-v4-pro /lustre/fsw/models/dsv4-pro /lustre/fsw/models/DeepSeek-V4-Pro; do - if [[ -d "$candidate" ]]; then - SELECTED_MODEL_PATH="$candidate" - break - fi - done - fi - export MODEL_PATH="${SELECTED_MODEL_PATH:-/lustre/fsw/models/deepseek-v4-pro}" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "bf16" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B" - export SRT_SLURM_MODEL_PREFIX="qwen3.5" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-FP8" - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4" - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/GLM-5-FP8" - export SRT_SLURM_MODEL_PREFIX="glm5-fp8" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/GLM-5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="glm5-fp4" -elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5" - export SRT_SLURM_MODEL_PREFIX="kimik2.5" -elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" -elif [[ $MODEL_PREFIX == "gptoss" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/gpt-oss-120b" - export SRT_SLURM_MODEL_PREFIX="gptoss" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - # Day-zero: MiniMax-M3-MXFP8 is not in the SRE-staged /lustre/fsw/models - # tree (root-owned); it lives in the sa-shared-writable gharunners tree. - export MODEL_PATH="/lustre/fsw/gharunners/models/MiniMax-M3-MXFP8" - export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then - # NVFP4 checkpoint, pre-staged on the b200-dgxc scratch tree. - export MODEL_PATH="/scratch/fsw/models/MiniMax-M3-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" -else - echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION" - echo "Available models under /lustre/fsw/models:" - ls -la /lustre/fsw/models +if ! RESOLVED_MODEL_ENV=$(python3 "$GITHUB_WORKSPACE/utils/resolve_runner_model.py" \ + --runner-config "$GITHUB_WORKSPACE/configs/runners.yaml" \ + --runner-label "cluster:b200-dgxc" \ + --model-prefix "$MODEL_PREFIX" \ + --precision "$PRECISION"); then exit 1 fi +eval "$RESOLVED_MODEL_ENV" export AIPERF_MMAP_CACHE_HOST_PATH="/lustre/fsw/gharunners/aiperf-cache" diff --git a/utils/matrix_logic/test_resolve_runner_model.py b/utils/matrix_logic/test_resolve_runner_model.py new file mode 100644 index 0000000000..ab7dd9038c --- /dev/null +++ b/utils/matrix_logic/test_resolve_runner_model.py @@ -0,0 +1,94 @@ +"""Tests for cluster-local model path resolution.""" + +import pytest + +from utils.resolve_runner_model import resolve_runner_model, shell_exports + + +def runner_config(model_paths, allow_override=False): + """Build a minimal runner model configuration.""" + return { + "models": { + "cluster:b200-dgxc": { + "dsv4": { + "fp4": { + "model-paths": model_paths, + "srt-slurm-model-prefix": "deepseek-v4-pro", + "allow-model-path-override": allow_override, + } + } + } + } + } + + +def test_resolve_uses_first_existing_configured_path(): + """The first existing configured path wins.""" + config = runner_config(["/models/preferred", "/models/fallback"]) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dsv4", + "fp4", + path_is_dir=lambda path: path == "/models/fallback", + ) + + assert result == ("/models/fallback", "deepseek-v4-pro") + + +def test_resolve_falls_back_to_first_path_when_none_exist(): + """The first path remains the deterministic fallback.""" + config = runner_config(["/models/preferred", "/models/fallback"]) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dsv4", + "fp4", + path_is_dir=lambda _path: False, + ) + + assert result == ("/models/preferred", "deepseek-v4-pro") + + +def test_resolve_honors_allowed_existing_override(): + """An allowed, existing MODEL_PATH override takes precedence.""" + config = runner_config(["/models/preferred"], allow_override=True) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dsv4", + "fp4", + current_model_path="/models/operator-override", + path_is_dir=lambda path: path == "/models/operator-override", + ) + + assert result == ("/models/operator-override", "deepseek-v4-pro") + + +def test_resolve_rejects_unsupported_combination(): + """Missing model combinations fail with the full lookup key.""" + config = runner_config(["/models/preferred"]) + + with pytest.raises(ValueError) as exc_info: + resolve_runner_model( + config, + "cluster:b200-dgxc", + "dsr1", + "fp8", + path_is_dir=lambda _path: False, + ) + + assert "cluster:b200-dgxc/dsr1/fp8" in str(exc_info.value) + + +def test_shell_exports_quote_values(): + """Shell output quotes values instead of interpolating them.""" + exports = shell_exports("/models/path with spaces", "model;alias") + + assert exports == ( + "export MODEL_PATH='/models/path with spaces'\n" + "export SRT_SLURM_MODEL_PREFIX='model;alias'" + ) diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 3d25d03eef..7eeffc1644 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -1359,6 +1359,84 @@ def test_hardware_gpus_per_node_must_be_positive(self): validate_runner_config(config) assert "gpus-per-node" in str(exc_info.value) + def test_valid_runner_model_mapping(self): + config = { + "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "dsv4": { + "fp4": { + "model-paths": [ + "/lustre/fsw/models/deepseek-v4-pro", + "/lustre/fsw/models/dsv4-pro", + ], + "srt-slurm-model-prefix": "deepseek-v4-pro", + "allow-model-path-override": True, + } + } + } + }, + } + + assert validate_runner_config(config) == config + + def test_runner_model_mapping_requires_known_label(self): + config = { + "labels": {"b200": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "dsr1": { + "fp8": { + "model-paths": ["/lustre/fsw/models/dsr1"], + "srt-slurm-model-prefix": "dsr1-fp8", + } + } + } + }, + } + + with pytest.raises(ValueError, match="unknown labels"): + validate_runner_config(config) + + def test_runner_model_paths_must_be_absolute(self): + config = { + "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "dsr1": { + "fp8": { + "model-paths": ["relative/model/path"], + "srt-slurm-model-prefix": "dsr1-fp8", + } + } + } + }, + } + + with pytest.raises(ValueError, match="must be absolute"): + validate_runner_config(config) + + def test_runner_model_paths_must_be_unique(self): + config = { + "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "dsr1": { + "fp8": { + "model-paths": [ + "/lustre/fsw/models/dsr1", + "/lustre/fsw/models/dsr1", + ], + "srt-slurm-model-prefix": "dsr1-fp8", + } + } + } + }, + } + + with pytest.raises(ValueError, match="must not contain duplicates"): + validate_runner_config(config) + # ============================================================================= # Test changelog entry validation diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 9f19572f08..69a6925011 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -831,12 +831,56 @@ class RunnerHardwareConfig(BaseModel): ) +class RunnerModelConfig(BaseModel): + """Cluster-local model paths and aliases used by runner launchers.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + model_paths: List[str] = Field(alias='model-paths', min_length=1) + srt_slurm_model_prefix: str = Field( + alias='srt-slurm-model-prefix', min_length=1 + ) + allow_model_path_override: bool = Field( + alias='allow-model-path-override', default=False + ) + + @field_validator('model_paths') + @classmethod + def validate_model_paths(cls, model_paths: List[str]) -> List[str]: + """Require unique absolute paths that are safe to export to a shell.""" + if len(model_paths) != len(set(model_paths)): + raise ValueError("'model-paths' must not contain duplicates") + for model_path in model_paths: + if not model_path.startswith('/'): + raise ValueError( + f"Runner model path must be absolute: {model_path}" + ) + if '\n' in model_path or '\t' in model_path: + raise ValueError( + "Runner model paths must not contain tabs or newlines" + ) + return model_paths + + class RunnerConfig(BaseModel): """Top-level runner configuration file.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) labels: Dict[str, List[str]] hardware: Dict[str, RunnerHardwareConfig] = Field(default_factory=dict) + models: Dict[str, Dict[str, Dict[str, RunnerModelConfig]]] = Field( + default_factory=dict + ) + + @model_validator(mode='after') + def validate_model_runner_labels(self): + """Require model mappings to reference a declared runner label.""" + unknown_labels = sorted(set(self.models) - set(self.labels)) + if unknown_labels: + raise ValueError( + "Runner model mappings reference unknown labels: " + + ", ".join(unknown_labels) + ) + return self def validate_runner_config(runner_configs: dict) -> dict: diff --git a/utils/resolve_runner_model.py b/utils/resolve_runner_model.py new file mode 100644 index 0000000000..5e7095cf53 --- /dev/null +++ b/utils/resolve_runner_model.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Resolve cluster-local model paths from configs/runners.yaml.""" + +import argparse +import os +from pathlib import Path +import shlex +import sys +from typing import Callable, Optional + +sys.path.insert(0, str(Path(__file__).resolve().parent / "matrix_logic")) + +from validation import load_runner_file + + +def resolve_runner_model( + runner_config: dict, + runner_label: str, + model_prefix: str, + precision: str, + current_model_path: Optional[str] = None, + path_is_dir: Callable[[str], bool] = os.path.isdir, +) -> tuple[str, str]: + """Return the model path and srt-slurm alias for a runner/model pair.""" + models = runner_config.get("models", {}) + try: + model_config = models[runner_label][model_prefix][precision] + except KeyError as error: + raise ValueError( + "Unsupported runner/model/precision combination: " + f"{runner_label}/{model_prefix}/{precision}" + ) from error + + configured_paths = model_config["model-paths"] + if ( + model_config.get("allow-model-path-override", False) + and current_model_path + and path_is_dir(current_model_path) + ): + model_path = current_model_path + else: + model_path = next( + (path for path in configured_paths if path_is_dir(path)), + configured_paths[0], + ) + + return model_path, model_config["srt-slurm-model-prefix"] + + +def shell_exports(model_path: str, srt_slurm_model_prefix: str) -> str: + """Render resolved values as shell-safe export statements.""" + return "\n".join( + [ + f"export MODEL_PATH={shlex.quote(model_path)}", + "export SRT_SLURM_MODEL_PREFIX=" + f"{shlex.quote(srt_slurm_model_prefix)}", + ] + ) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--runner-config", + default="configs/runners.yaml", + help="Path to runners.yaml", + ) + parser.add_argument("--runner-label", required=True) + parser.add_argument("--model-prefix", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument( + "--current-model-path", + default=os.environ.get("MODEL_PATH"), + help="Existing MODEL_PATH override, when the mapping allows it", + ) + return parser.parse_args() + + +def main() -> int: + """Resolve and print shell exports for a runner model mapping.""" + args = parse_args() + try: + runner_config = load_runner_file(args.runner_config) + model_path, srt_slurm_model_prefix = resolve_runner_model( + runner_config, + args.runner_label, + args.model_prefix, + args.precision, + args.current_model_path, + ) + except ValueError as error: + print(f"Error: {error}", file=sys.stderr) + return 1 + + print(shell_exports(model_path, srt_slurm_model_prefix)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 19a7c12a8bd05d15350e6d7c11ce6ed0587a0f88 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 28 Jul 2026 09:31:32 -0500 Subject: [PATCH 2/2] refactor(runners): centralize all model path mappings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate the remaining B300, GB200, GB300, H100, and H200 launcher lookup tables into configs/runners.yaml. Generalize model resolution for framework overrides, optional aliases, and portable fallbacks, and add coverage for every affected NVIDIA multi-node config. 中文：将剩余的 B300、GB200、GB300、H100 和 H200 启动器模型查找表统一迁移到 configs/runners.yaml。扩展模型解析能力以支持框架级覆盖、可选别名及便携式回退路径，并为所有受影响的 NVIDIA 多节点配置补充覆盖测试。 --- configs/runners.yaml | 384 ++++++++++++++---- runners/launch_b200-dgxc.sh | 11 +- runners/launch_b300-nv.sh | 45 +- runners/launch_gb200-nv.sh | 103 +---- runners/launch_gb300-nv.sh | 70 +--- runners/launch_h100-dgxc-slurm.sh | 26 +- runners/launch_h200-dgxc-slurm.sh | 26 +- runners/runner_model_utils.sh | 23 ++ .../matrix_logic/test_resolve_runner_model.py | 211 +++++++++- utils/matrix_logic/test_validation.py | 54 +-- utils/matrix_logic/validation.py | 15 +- utils/resolve_runner_model.py | 81 +++- 12 files changed, 666 insertions(+), 383 deletions(-) create mode 100644 runners/runner_model_utils.sh diff --git a/configs/runners.yaml b/configs/runners.yaml index 4235e429bd..7cdd53ccb5 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -334,79 +334,313 @@ models: # Cluster-local checkpoints used by launchers instead of downloading the # portable HuggingFace model IDs in the benchmark configs. When multiple # paths are listed, the first existing directory wins and the first entry is - # the deterministic fallback. + # the deterministic fallback. Framework-specific mappings take precedence + # over the default mapping. cluster:b200-dgxc: - dsr1: - fp4: - # Per-node /raid is not populated reliably across the cluster. - model-paths: - - /scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2 - srt-slurm-model-prefix: dsr1 - fp8: - model-paths: - - /lustre/fsw/models/dsr1-0528-fp8 - srt-slurm-model-prefix: dsr1-fp8 - dsv4: - fp4: - model-paths: - - /lustre/fsw/models/deepseek-v4-pro - - /lustre/fsw/models/dsv4-pro - - /lustre/fsw/models/DeepSeek-V4-Pro - srt-slurm-model-prefix: deepseek-v4-pro - # Preserve the operator override supported by the previous launcher. - allow-model-path-override: true - qwen3.5: - bf16: - model-paths: - - /lustre/fsw/models/Qwen3.5-397B-A17B - srt-slurm-model-prefix: qwen3.5 - fp8: - model-paths: - - /lustre/fsw/models/Qwen3.5-397B-A17B-FP8 - srt-slurm-model-prefix: qwen3.5-fp8 - fp4: - model-paths: - - /lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4 - srt-slurm-model-prefix: qwen3.5-fp4 - glm5: - fp8: - model-paths: - - /lustre/fsw/models/GLM-5-FP8 - srt-slurm-model-prefix: glm5-fp8 - fp4: - model-paths: - - /lustre/fsw/models/GLM-5-NVFP4 - srt-slurm-model-prefix: glm5-fp4 - kimik2.5: - int4: - model-paths: - - /lustre/fsw/models/Kimi-K2.5 - srt-slurm-model-prefix: kimik2.5 - fp4: - model-paths: - - /lustre/fsw/models/Kimi-K2.5-NVFP4 - srt-slurm-model-prefix: kimik2.5-fp4 - minimaxm2.5: - fp8: - model-paths: - - /lustre/fsw/models/MiniMax-M2.5 - srt-slurm-model-prefix: minimax-m2.5-fp8 - fp4: - model-paths: - - /lustre/fsw/models/MiniMax-M2.5-NVFP4 - srt-slurm-model-prefix: minimax-m2.5-nvfp4 - gptoss: - fp4: - model-paths: - - /lustre/fsw/models/gpt-oss-120b - srt-slurm-model-prefix: gptoss - minimaxm3: - fp8: - # Day-zero model staged in the sa-shared-writable gharunners tree. - model-paths: - - /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 - srt-slurm-model-prefix: minimax-m3-mxfp8 - fp4: - model-paths: - - /scratch/fsw/models/MiniMax-M3-NVFP4 - srt-slurm-model-prefix: minimax-m3-nvfp4 + default: + dsr1: + fp4: + # Per-node /raid is not populated reliably across the cluster. + model-paths: + - /scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /lustre/fsw/models/dsr1-0528-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dsv4: + fp4: + model-paths: + - /lustre/fsw/models/deepseek-v4-pro + - /lustre/fsw/models/dsv4-pro + - /lustre/fsw/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-v4-pro + # Preserve the operator override supported by the previous launcher. + allow-model-path-override: true + qwen3.5: + bf16: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B + srt-slurm-model-prefix: qwen3.5 + fp8: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B-FP8 + srt-slurm-model-prefix: qwen3.5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4 + srt-slurm-model-prefix: qwen3.5-fp4 + glm5: + fp8: + model-paths: + - /lustre/fsw/models/GLM-5-FP8 + srt-slurm-model-prefix: glm5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/GLM-5-NVFP4 + srt-slurm-model-prefix: glm5-fp4 + kimik2.5: + int4: + model-paths: + - /lustre/fsw/models/Kimi-K2.5 + srt-slurm-model-prefix: kimik2.5 + fp4: + model-paths: + - /lustre/fsw/models/Kimi-K2.5-NVFP4 + srt-slurm-model-prefix: kimik2.5-fp4 + minimaxm2.5: + fp8: + model-paths: + - /lustre/fsw/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + gptoss: + fp4: + model-paths: + - /lustre/fsw/models/gpt-oss-120b + srt-slurm-model-prefix: gptoss + minimaxm3: + fp8: + # Day-zero model staged in the sa-shared-writable gharunners tree. + model-paths: + - /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: minimax-m3-mxfp8 + fp4: + model-paths: + - /scratch/fsw/models/MiniMax-M3-NVFP4 + srt-slurm-model-prefix: minimax-m3-nvfp4 + cluster:b300-nv: + default: + dsr1: + fp4: + model-paths: + - /data/models/dsr1-fp4 + served-model-name: deepseek-r1-fp4 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /data/models/dsr1-fp8 + served-model-name: deepseek-r1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dynamo-vllm: + dsv4: + fp4: + model-paths: + - /data/models/dsv4-pro + - /data/models/deepseek-v4-pro + - /data/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-v4-pro + allow-model-path-override: true + minimaxm2.5: + fp4: + model-paths: + - /data/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + fp8: + model-paths: + - /data/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + minimaxm3: + fp4: + model-paths: + - /scratch/models/MiniMax-M3-NVFP4 + srt-slurm-model-prefix: nvidia/MiniMax-M3-NVFP4 + fp8: + model-paths: + - /data/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: MiniMaxAI/MiniMax-M3-MXFP8 + cluster:gb200-nv: + llmd-vllm: + dsv4: + fp4: + model-paths: + - /mnt/numa1/models/DeepSeek-V4-Pro + model-name: deepseek-ai/DeepSeek-V4-Pro + dynamo-sglang: + dsr1: + fp8: + model-paths: + - /mnt/lustre01/models/deepseek-r1-0528 + srt-slurm-model-prefix: dsr1-fp8 + fp4: + model-paths: + - /mnt/lustre01/models/deepseek-r1-0528-fp4-v2/ + srt-slurm-model-prefix: dsr1-fp4 + dsv4: + fp4: + model-paths: + - /mnt/lustre01/models/deepseek-v4-pro + srt-slurm-model-prefix: deepseek-v4-pro + glm5.1: + fp4: + model-paths: + - /mnt/lustre01/models/GLM-5.1-NVFP4 + srt-slurm-model-prefix: glm-5-fp4 + fp8: + model-paths: + - /mnt/lustre01/models/GLM-5.1-FP8 + srt-slurm-model-prefix: glm-5.1-fp8 + qwen3.5: + fp8: + model-paths: + - /mnt/lustre01/models/Qwen3.5-397B-A17B-FP8 + srt-slurm-model-prefix: qwen3.5-fp8 + dynamo-trt: + gptoss: + default: + model-paths: + - /mnt/lustre01/models/gpt-oss-120b + served-model-name: gpt-oss-120b + dsr1: + fp4: + model-paths: + - /mnt/numa1/models/DeepSeek-R1-0528-NVFP4-v2 + served-model-name: deepseek-r1-fp4 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /mnt/numa1/models/DeepSeek-R1-0528 + served-model-name: deepseek-r1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + kimik2.5: + fp4: + model-paths: + - /mnt/lustre01/models/kimi-k2.5-nvfp4 + served-model-name: kimi-k2.5-nvfp4 + srt-slurm-model-prefix: nvidia/Kimi-K2.5-NVFP4 + glm5: + fp4: + model-paths: + - /mnt/lustre01/slurm-shared/glm-model/GLM-5-NVFP4 + served-model-name: glm-5-nvfp4 + srt-slurm-model-prefix: nvidia/GLM-5-NVFP4 + dynamo-vllm: + kimik2.5: + fp4: + model-paths: + - /mnt/lustre01/models/kimi-k2.5-nvfp4 + srt-slurm-model-prefix: kimi-k2.5-nvfp4 + dsv4: + fp4: + # Compute-visible NVFP4 checkpoint; the lowercase sibling is FP8. + model-paths: + - /mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/ + srt-slurm-model-prefix: deepseek-v4-pro + minimaxm2.5: + fp4: + model-paths: + - /mnt/lustre01/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + fp8: + model-paths: + - /mnt/lustre01/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + minimaxm3: + fp8: + model-paths: + - /mnt/lustre01/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: minimax-m3-mxfp8 + cluster:gb300-nv: + default: + dsr1: + fp4: + model-paths: + - /scratch/models/DeepSeek-R1-0528-NVFP4-v2 + served-model-name: deepseek-r1-fp4 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /scratch/models/DeepSeek-R1-0528 + served-model-name: deepseek-r1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dsv4: + fp4: + # Compute-node-local path; srtctl preflight must be skipped. + model-paths: + - /scratch/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-v4-pro + glm5.1: + fp4: + model-paths: + - /scratch/models/GLM-5.1-NVFP4 + srt-slurm-model-prefix: glm-5-fp4 + glm5: + fp4: + model-paths: + - /scratch/models/GLM-5-NVFP4 + srt-slurm-model-prefix: glm-5-fp4 + fp8: + model-paths: + - /scratch/models/GLM-5-FP8 + srt-slurm-model-prefix: glm-5-fp8 + minimaxm2.5: + fp4: + model-paths: + - /data/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + fp8: + model-paths: + - /data/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + minimaxm3: + fp8: + model-paths: + - /data/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: minimax-m3-mxfp8 + kimik2.5: + fp4: + model-paths: + - /scratch/models/Kimi-K2.5-NVFP4 + srt-slurm-model-prefix: nvidia/Kimi-K2.5-NVFP4 + qwen3.5: + fp4: + model-paths: + - /scratch/models/Qwen3.5-397B-A17B-NVFP4 + srt-slurm-model-prefix: qwen3.5-fp4 + fp8: + model-paths: + - /scratch/models/Qwen3.5-397B-A17B-FP8 + srt-slurm-model-prefix: qwen3.5-fp8 + dynamo-trt: + dsv4: + fp4: + model-paths: + - /scratch/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-ai/DeepSeek-V4-Pro + glm5: + fp4: + model-paths: + - /scratch/models/GLM-5-NVFP4 + served-model-name: glm-5-nvfp4 + srt-slurm-model-prefix: nvidia/GLM-5-NVFP4 + cluster:h100-dgxc: + dynamo-sglang: + dsr1: + fp8: + model-paths: + - /mnt/nfs/lustre/models/dsr1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dynamo-trt: + dsr1: + fp8: + model-paths: + - /mnt/nfs/lustre/models/dsr1-fp8 + served-model-name: DeepSeek-R1-0528 + srt-slurm-model-prefix: DeepSeek-R1-0528 + cluster:h200-dgxc: + dynamo-sglang: + dsr1: + fp8: + model-paths: + - /models/DeepSeek-R1-0528 + srt-slurm-model-prefix: dsr1-fp8 + dynamo-trt: + dsr1: + fp8: + model-paths: + - /models/DeepSeek-R1-0528 + served-model-name: DeepSeek-R1-0528 + srt-slurm-model-prefix: DeepSeek-R1-0528 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index d104b8e93e..d86fe76d29 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -6,14 +6,9 @@ SLURM_ACCOUNT="benchmark" set -x -if ! RESOLVED_MODEL_ENV=$(python3 "$GITHUB_WORKSPACE/utils/resolve_runner_model.py" \ - --runner-config "$GITHUB_WORKSPACE/configs/runners.yaml" \ - --runner-label "cluster:b200-dgxc" \ - --model-prefix "$MODEL_PREFIX" \ - --precision "$PRECISION"); then - exit 1 -fi -eval "$RESOLVED_MODEL_ENV" +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" +resolve_runner_model_config "cluster:b200-dgxc" || exit 1 export AIPERF_MMAP_CACHE_HOST_PATH="/lustre/fsw/gharunners/aiperf-cache" diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 098e984be5..6b0e30215c 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -8,6 +8,9 @@ MINIMAX_M3_SLURM_EXCLUDED_NODELIST="${MINIMAX_M3_SLURM_EXCLUDED_NODELIST-b300-01 set -x +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + if [[ "$IS_MULTINODE" == "true" ]]; then # Validate framework @@ -16,47 +19,7 @@ if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" && $FRAMEWORK exit 1 fi -# MODEL_PATH: Override with pre-downloaded paths on B300 runner -# The yaml files specify HuggingFace model IDs for portability, but we use -# local paths to avoid repeated downloading on the shared B300 cluster. -if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/data/models/dsr1-fp4" - export SERVED_MODEL_NAME="deepseek-r1-fp4" - export SRT_SLURM_MODEL_PREFIX="dsr1" -elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/data/models/dsr1-fp8" - export SERVED_MODEL_NAME="deepseek-r1-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then - SELECTED_MODEL_PATH="" - if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then - SELECTED_MODEL_PATH="$MODEL_PATH" - else - for candidate in /data/models/dsv4-pro /data/models/deepseek-v4-pro /data/models/DeepSeek-V4-Pro; do - if [[ -d "$candidate" ]]; then - SELECTED_MODEL_PATH="$candidate" - break - fi - done - fi - export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/data/models/MiniMax-M2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/data/models/MiniMax-M2.5" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/scratch/models/MiniMax-M3-NVFP4" - export SRT_SLURM_MODEL_PREFIX="nvidia/MiniMax-M3-NVFP4" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" - export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" -else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" - exit 1 -fi +resolve_runner_model_config "cluster:b300-nv" || exit 1 echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 426cd2dd87..da64129f87 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -5,19 +5,15 @@ set -x source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" export SLURM_PARTITION="batch" export SLURM_ACCOUNT="benchmark" SQUASH_DIR="/mnt/lustre01/users-public/sa-shared" if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then - if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then - export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" - export MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro" - else - echo "Unsupported MODEL_PREFIX/PRECISION for llmd-vllm on GB200: $MODEL_PREFIX/$PRECISION" >&2 - exit 1 - fi + resolve_runner_model_config "cluster:gb200-nv" || exit 1 SQUASH_FILE="${SQUASH_DIR}/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" @@ -87,96 +83,15 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then exit 0 fi -# MODEL_PATH: Override with pre-downloaded paths on GB200 runner -# The yaml files specify HuggingFace model IDs for portability, but we use -# local paths to avoid repeated downloading on the shared GB200 cluster. if [[ $FRAMEWORK == "dynamo-sglang" ]]; then export CONFIG_DIR="/mnt/lustre01/artifacts/sglang-configs/1k1k" - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528-fp4-v2/" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp4" - elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # Lustre-resident weights staged on the GB200 external cluster. - # SRT_SLURM_MODEL_PREFIX matches the model.path alias in our - # DSV4 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/deepseek-v4-pro" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" - elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-NVFP4" - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" - elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-FP8" - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" - elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-NVFP4" - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" - elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5.1-fp8") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-FP8" - export SRT_SLURM_MODEL_PREFIX="glm-5.1-fp8" - else - export MODEL_PATH=$MODEL - fi -elif [[ $FRAMEWORK == "dynamo-trt" ]]; then - if [[ $MODEL_PREFIX == "gptoss" ]]; then - export MODEL_PATH="/mnt/lustre01/models/gpt-oss-120b" - export SERVED_MODEL_NAME="gpt-oss-120b" - elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/numa1/models/DeepSeek-R1-0528-NVFP4-v2" - export SERVED_MODEL_NAME="deepseek-r1-fp4" - export SRT_SLURM_MODEL_PREFIX="dsr1" - elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/numa1/models/DeepSeek-R1-0528" - export SERVED_MODEL_NAME="deepseek-r1-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" - export SERVED_MODEL_NAME="kimi-k2.5-nvfp4" - export SRT_SLURM_MODEL_PREFIX="nvidia/Kimi-K2.5-NVFP4" - elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias - # ("nvidia/GLM-5-NVFP4") in the upstream GLM5 trtllm_dynamo recipes. - export MODEL_PATH="/mnt/lustre01/slurm-shared/glm-model/GLM-5-NVFP4" - export SERVED_MODEL_NAME="glm-5-nvfp4" - export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5-NVFP4" - else - echo "Unsupported model prefix: $MODEL_PREFIX. Supported prefixes are: gptoss, dsr1, kimik2.5, or glm5" - exit 1 - fi -elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then - if [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" - export SRT_SLURM_MODEL_PREFIX="kimi-k2.5-nvfp4" - elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # The FP4 checkpoint is staged on compute-visible Lustre. The former - # /mnt/numa1 path is no longer present on watchtower compute nodes; - # the lowercase Lustre sibling is the FP8 checkpoint, so keep the - # NVFP4 path explicit here. - export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" - elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" - elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" - elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-MXFP8" - export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" - else - echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp8" - exit 1 - fi + resolve_runner_model_config "cluster:gb200-nv" \ + --fallback-model-path "$MODEL" || exit 1 +elif [[ $FRAMEWORK == "dynamo-trt" || $FRAMEWORK == "dynamo-vllm" ]]; then + resolve_runner_model_config "cluster:gb200-nv" || exit 1 else - export MODEL_PATH=$MODEL + resolve_runner_model_config "cluster:gb200-nv" \ + --fallback-model-path "$MODEL" || exit 1 fi NGINX_IMAGE="nginx:1.27.4" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index e5a8d059b2..34e87f683f 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -4,6 +4,9 @@ set -exo pipefail +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + export SLURM_PARTITION="batch_1" export SLURM_ACCOUNT="benchmark" export ENROOT_ROOTFS_WRITABLE=1 @@ -31,69 +34,7 @@ mkdir -p "$HF_HUB_CACHE_HOST_PATH" export DYNAMO_WHEELS_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/dynamo-wheels" mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH" -export MODEL_PATH=$MODEL - -if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export SERVED_MODEL_NAME="deepseek-r1-fp4" - export MODEL_PATH=/scratch/models/DeepSeek-R1-0528-NVFP4-v2 - export SRT_SLURM_MODEL_PREFIX="dsr1" -elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export SERVED_MODEL_NAME="deepseek-r1-fp8" - export MODEL_PATH=/scratch/models/DeepSeek-R1-0528 - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # Use the node-local /scratch SSD for the 806 GB DSv4-Pro - # checkpoint. Faster than the Vast NFS path, but this dir only - # exists on compute nodes — the GHA runner pod's view does NOT - # have /scratch/models, so srtctl preflight (which stats the path - # from the runner pod) may fail with "Model alias resolved to - # /scratch/models/DeepSeek-V4-Pro, but that path is unavailable." - # If that happens, the next step is either to (a) patch srt-slurm - # to add a skip_model_preflight recipe field, or (b) stub a - # symlink on the runner pod that points at the NFS copy. - export MODEL_PATH=/scratch/models/DeepSeek-V4-Pro - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-trt" ]]; then - export SERVED_MODEL_NAME="glm-5-nvfp4" - export MODEL_PATH=/scratch/models/GLM-5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5-NVFP4" -elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH=/scratch/models/GLM-5.1-NVFP4 - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH=/scratch/models/GLM-5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH=/scratch/models/GLM-5-FP8 - export SRT_SLURM_MODEL_PREFIX="glm-5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH=/data/models/MiniMax-M2.5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH=/data/models/MiniMax-M2.5 - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - export MODEL_PATH=/data/models/MiniMax-M3-MXFP8 - export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" -elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH=/scratch/models/Kimi-K2.5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="nvidia/Kimi-K2.5-NVFP4" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our - # Qwen3.5 sglang recipes (qwen3.5-fp4). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then - # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our - # Qwen3.5 sglang recipes (qwen3.5-fp8). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-FP8 - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" -else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" - exit 1 -fi +resolve_runner_model_config "cluster:gb300-nv" || exit 1 NGINX_IMAGE="nginx:1.27.4" @@ -223,9 +164,6 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.5" && $PRECISION mkdir -p recipes/vllm/kimi-k2.5-fp4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4" recipes/vllm/kimi-k2.5-fp4 elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 dynamo-trt recipes use the HuggingFace model ID as model.path, - # so override SRT_SLURM_MODEL_PREFIX to match the recipe's model path key. - SRT_SLURM_MODEL_PREFIX="deepseek-ai/DeepSeek-V4-Pro" git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 1334c95542..c9d948cfba 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -11,32 +11,16 @@ SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') set -x +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + if [[ "$IS_MULTINODE" == "true" ]]; then - # MODEL_PATH: Override with pre-downloaded paths on H100 runner - # The yaml files specify HuggingFace model IDs for portability, but we use - # local paths to avoid repeated downloading on the shared H100 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/nfs/lustre/models/dsr1-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - else - echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - elif [[ $FRAMEWORK == "dynamo-trt" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/nfs/lustre/models/dsr1-fp8" - export SERVED_MODEL_NAME="DeepSeek-R1-0528" - export SRT_SLURM_MODEL_PREFIX="DeepSeek-R1-0528" - else - echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - else + if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" ]]; then echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" exit 1 fi + resolve_runner_model_config "cluster:h100-dgxc" || exit 1 echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 00c6cc4977..3bf0cc5c66 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -7,32 +7,16 @@ SLURM_ACCOUNT="sa-shared" set -x +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + if [[ "$IS_MULTINODE" == "true" ]]; then - # MODEL_PATH: Override with pre-downloaded paths on H200 runner - # The yaml files specify HuggingFace model IDs for portability, but we use - # local paths to avoid repeated downloading on the shared H200 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/models/DeepSeek-R1-0528" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - else - echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - elif [[ $FRAMEWORK == "dynamo-trt" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/models/DeepSeek-R1-0528" - export SERVED_MODEL_NAME="DeepSeek-R1-0528" - export SRT_SLURM_MODEL_PREFIX="DeepSeek-R1-0528" - else - echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - else + if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" ]]; then echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" exit 1 fi + resolve_runner_model_config "cluster:h200-dgxc" || exit 1 echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" diff --git a/runners/runner_model_utils.sh b/runners/runner_model_utils.sh new file mode 100644 index 0000000000..a31cdccd4e --- /dev/null +++ b/runners/runner_model_utils.sh @@ -0,0 +1,23 @@ +#!/usr/bin/bash + +# Resolve a launcher's cluster-local model environment from configs/runners.yaml. +resolve_runner_model_config() { + local runner_label="$1" + shift + + local runner_utils_dir repo_root resolved_model_env + runner_utils_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + repo_root="$(cd "$runner_utils_dir/.." && pwd)" + + if ! resolved_model_env=$(python3 "$repo_root/utils/resolve_runner_model.py" \ + --runner-config "$repo_root/configs/runners.yaml" \ + --runner-label "$runner_label" \ + --framework "$FRAMEWORK" \ + --model-prefix "$MODEL_PREFIX" \ + --precision "$PRECISION" \ + "$@"); then + return 1 + fi + + eval "$resolved_model_env" +} diff --git a/utils/matrix_logic/test_resolve_runner_model.py b/utils/matrix_logic/test_resolve_runner_model.py index ab7dd9038c..bcaad81800 100644 --- a/utils/matrix_logic/test_resolve_runner_model.py +++ b/utils/matrix_logic/test_resolve_runner_model.py @@ -1,20 +1,27 @@ """Tests for cluster-local model path resolution.""" +from pathlib import Path + import pytest +from utils.matrix_logic.validation import load_config_files, load_runner_file from utils.resolve_runner_model import resolve_runner_model, shell_exports +REPO_ROOT = Path(__file__).resolve().parents[2] + def runner_config(model_paths, allow_override=False): """Build a minimal runner model configuration.""" return { "models": { "cluster:b200-dgxc": { - "dsv4": { - "fp4": { - "model-paths": model_paths, - "srt-slurm-model-prefix": "deepseek-v4-pro", - "allow-model-path-override": allow_override, + "default": { + "dsv4": { + "fp4": { + "model-paths": model_paths, + "srt-slurm-model-prefix": "deepseek-v4-pro", + "allow-model-path-override": allow_override, + } } } } @@ -29,12 +36,16 @@ def test_resolve_uses_first_existing_configured_path(): result = resolve_runner_model( config, "cluster:b200-dgxc", + "dynamo-vllm", "dsv4", "fp4", path_is_dir=lambda path: path == "/models/fallback", ) - assert result == ("/models/fallback", "deepseek-v4-pro") + assert result == { + "MODEL_PATH": "/models/fallback", + "SRT_SLURM_MODEL_PREFIX": "deepseek-v4-pro", + } def test_resolve_falls_back_to_first_path_when_none_exist(): @@ -44,12 +55,13 @@ def test_resolve_falls_back_to_first_path_when_none_exist(): result = resolve_runner_model( config, "cluster:b200-dgxc", + "dynamo-vllm", "dsv4", "fp4", path_is_dir=lambda _path: False, ) - assert result == ("/models/preferred", "deepseek-v4-pro") + assert result["MODEL_PATH"] == "/models/preferred" def test_resolve_honors_allowed_existing_override(): @@ -59,13 +71,87 @@ def test_resolve_honors_allowed_existing_override(): result = resolve_runner_model( config, "cluster:b200-dgxc", + "dynamo-vllm", "dsv4", "fp4", current_model_path="/models/operator-override", path_is_dir=lambda path: path == "/models/operator-override", ) - assert result == ("/models/operator-override", "deepseek-v4-pro") + assert result["MODEL_PATH"] == "/models/operator-override" + + +def test_resolve_prefers_framework_specific_mapping(): + """An exact framework mapping overrides the runner default.""" + config = runner_config(["/models/default"]) + config["models"]["cluster:b200-dgxc"]["dynamo-trt"] = { + "dsv4": { + "fp4": { + "model-paths": ["/models/trt"], + "srt-slurm-model-prefix": "deepseek-ai/DeepSeek-V4-Pro", + "served-model-name": "deepseek-v4-pro", + } + } + } + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-trt", + "dsv4", + "fp4", + path_is_dir=lambda _path: False, + ) + + assert result == { + "MODEL_PATH": "/models/trt", + "SRT_SLURM_MODEL_PREFIX": "deepseek-ai/DeepSeek-V4-Pro", + "SERVED_MODEL_NAME": "deepseek-v4-pro", + } + + +def test_resolve_supports_default_precision_mapping(): + """A default precision supports model mappings that ignore precision.""" + config = runner_config(["/models/default"]) + config["models"]["cluster:b200-dgxc"]["dynamo-trt"] = { + "gptoss": { + "default": { + "model-paths": ["/models/gpt-oss"], + "served-model-name": "gpt-oss-120b", + } + } + } + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-trt", + "gptoss", + "mxfp4", + path_is_dir=lambda _path: False, + ) + + assert result == { + "MODEL_PATH": "/models/gpt-oss", + "SERVED_MODEL_NAME": "gpt-oss-120b", + } + + +def test_resolve_supports_explicit_missing_mapping_fallback(): + """Callers may preserve a portable MODEL fallback for unmapped models.""" + config = runner_config(["/models/default"]) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-sglang", + "qwen", + "bf16", + fallback_model_path="Qwen/Qwen3", + path_is_dir=lambda _path: False, + ) + + assert result == {"MODEL_PATH": "Qwen/Qwen3"} def test_resolve_rejects_unsupported_combination(): @@ -76,19 +162,124 @@ def test_resolve_rejects_unsupported_combination(): resolve_runner_model( config, "cluster:b200-dgxc", + "dynamo-vllm", "dsr1", "fp8", path_is_dir=lambda _path: False, ) - assert "cluster:b200-dgxc/dsr1/fp8" in str(exc_info.value) + assert ( + "cluster:b200-dgxc/dynamo-vllm/dsr1/fp8" + in str(exc_info.value) + ) def test_shell_exports_quote_values(): """Shell output quotes values instead of interpolating them.""" - exports = shell_exports("/models/path with spaces", "model;alias") + exports = shell_exports( + { + "MODEL_PATH": "/models/path with spaces", + "SRT_SLURM_MODEL_PREFIX": "model;alias", + } + ) assert exports == ( "export MODEL_PATH='/models/path with spaces'\n" "export SRT_SLURM_MODEL_PREFIX='model;alias'" ) + + +def test_resolve_every_configured_runner_model_mapping(): + """Every runners.yaml model entry is reachable through the resolver.""" + config = load_runner_file(str(REPO_ROOT / "configs" / "runners.yaml")) + export_names = { + "srt-slurm-model-prefix": "SRT_SLURM_MODEL_PREFIX", + "served-model-name": "SERVED_MODEL_NAME", + "model-name": "MODEL_NAME", + } + + for runner_label, framework_mappings in config["models"].items(): + for framework_key, model_mappings in framework_mappings.items(): + framework = ( + "unconfigured-framework" + if framework_key == "default" + else framework_key + ) + for model_prefix, precision_mappings in model_mappings.items(): + for precision_key, model_config in precision_mappings.items(): + precision = ( + "unconfigured-precision" + if precision_key == "default" + else precision_key + ) + result = resolve_runner_model( + config, + runner_label, + framework, + model_prefix, + precision, + path_is_dir=lambda _path: False, + ) + + assert result["MODEL_PATH"] == model_config["model-paths"][0] + for config_name, export_name in export_names.items(): + if config_name in model_config: + assert result[export_name] == model_config[config_name] + + +def test_every_nvidia_multinode_runner_model_resolves(): + """Every multinode config routed to a migrated launcher is supported.""" + runner_config = load_runner_file( + str(REPO_ROOT / "configs" / "runners.yaml") + ) + master_config = load_config_files( + [str(REPO_ROOT / "configs" / "nvidia-master.yaml")] + ) + cluster_by_runner = { + "b200": "cluster:b200-dgxc", + "b200-dsv4": "cluster:b200-dgxc", + "b200-multinode": "cluster:b200-dgxc", + "cluster:b200-dgxc": "cluster:b200-dgxc", + "b300": "cluster:b300-nv", + "b300-p1": "cluster:b300-nv", + "cluster:b300-nv": "cluster:b300-nv", + "gb200": "cluster:gb200-nv", + "cluster:gb200-nv": "cluster:gb200-nv", + "gb300": "cluster:gb300-nv", + "gb300-nv": "cluster:gb300-nv", + "cluster:gb300-nv": "cluster:gb300-nv", + "h100-multinode": "cluster:h100-dgxc", + "cluster:h100-dgxc": "cluster:h100-dgxc", + "h200-multinode": "cluster:h200-dgxc", + "cluster:h200-dgxc": "cluster:h200-dgxc", + } + strict_gb200_frameworks = {"llmd-vllm", "dynamo-trt", "dynamo-vllm"} + unsupported = [] + + for config_name, benchmark in master_config.items(): + if not benchmark.get("multinode", False): + continue + runner_label = cluster_by_runner.get(benchmark["runner"]) + if runner_label is None: + continue + fallback_model_path = None + if ( + runner_label == "cluster:gb200-nv" + and benchmark["framework"] not in strict_gb200_frameworks + ): + fallback_model_path = benchmark["model"] + + try: + resolve_runner_model( + runner_config, + runner_label, + benchmark["framework"], + benchmark["model-prefix"], + benchmark["precision"], + fallback_model_path=fallback_model_path, + path_is_dir=lambda _path: False, + ) + except ValueError as error: + unsupported.append(f"{config_name}: {error}") + + assert not unsupported, "\n".join(unsupported) diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 7eeffc1644..32d24a617c 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -1364,14 +1364,16 @@ def test_valid_runner_model_mapping(self): "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, "models": { "cluster:b200-dgxc": { - "dsv4": { - "fp4": { - "model-paths": [ - "/lustre/fsw/models/deepseek-v4-pro", - "/lustre/fsw/models/dsv4-pro", - ], - "srt-slurm-model-prefix": "deepseek-v4-pro", - "allow-model-path-override": True, + "dynamo-vllm": { + "dsv4": { + "fp4": { + "model-paths": [ + "/lustre/fsw/models/deepseek-v4-pro", + "/lustre/fsw/models/dsv4-pro", + ], + "srt-slurm-model-prefix": "deepseek-v4-pro", + "allow-model-path-override": True, + } } } } @@ -1385,10 +1387,12 @@ def test_runner_model_mapping_requires_known_label(self): "labels": {"b200": ["b200-dgxc_0"]}, "models": { "cluster:b200-dgxc": { - "dsr1": { - "fp8": { - "model-paths": ["/lustre/fsw/models/dsr1"], - "srt-slurm-model-prefix": "dsr1-fp8", + "default": { + "dsr1": { + "fp8": { + "model-paths": ["/lustre/fsw/models/dsr1"], + "srt-slurm-model-prefix": "dsr1-fp8", + } } } } @@ -1403,10 +1407,12 @@ def test_runner_model_paths_must_be_absolute(self): "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, "models": { "cluster:b200-dgxc": { - "dsr1": { - "fp8": { - "model-paths": ["relative/model/path"], - "srt-slurm-model-prefix": "dsr1-fp8", + "default": { + "dsr1": { + "fp8": { + "model-paths": ["relative/model/path"], + "srt-slurm-model-prefix": "dsr1-fp8", + } } } } @@ -1421,13 +1427,15 @@ def test_runner_model_paths_must_be_unique(self): "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, "models": { "cluster:b200-dgxc": { - "dsr1": { - "fp8": { - "model-paths": [ - "/lustre/fsw/models/dsr1", - "/lustre/fsw/models/dsr1", - ], - "srt-slurm-model-prefix": "dsr1-fp8", + "default": { + "dsr1": { + "fp8": { + "model-paths": [ + "/lustre/fsw/models/dsr1", + "/lustre/fsw/models/dsr1", + ], + "srt-slurm-model-prefix": "dsr1-fp8", + } } } } diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 69a6925011..d9e784f55f 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -836,8 +836,14 @@ class RunnerModelConfig(BaseModel): model_config = ConfigDict(extra='forbid', populate_by_name=True) model_paths: List[str] = Field(alias='model-paths', min_length=1) - srt_slurm_model_prefix: str = Field( - alias='srt-slurm-model-prefix', min_length=1 + srt_slurm_model_prefix: Optional[str] = Field( + alias='srt-slurm-model-prefix', default=None, min_length=1 + ) + served_model_name: Optional[str] = Field( + alias='served-model-name', default=None, min_length=1 + ) + model_name: Optional[str] = Field( + alias='model-name', default=None, min_length=1 ) allow_model_path_override: bool = Field( alias='allow-model-path-override', default=False @@ -867,7 +873,10 @@ class RunnerConfig(BaseModel): labels: Dict[str, List[str]] hardware: Dict[str, RunnerHardwareConfig] = Field(default_factory=dict) - models: Dict[str, Dict[str, Dict[str, RunnerModelConfig]]] = Field( + models: Dict[ + str, + Dict[str, Dict[str, Dict[str, RunnerModelConfig]]] + ] = Field( default_factory=dict ) diff --git a/utils/resolve_runner_model.py b/utils/resolve_runner_model.py index 5e7095cf53..9dcc607e7f 100644 --- a/utils/resolve_runner_model.py +++ b/utils/resolve_runner_model.py @@ -3,33 +3,58 @@ import argparse import os -from pathlib import Path import shlex import sys from typing import Callable, Optional -sys.path.insert(0, str(Path(__file__).resolve().parent / "matrix_logic")) +import yaml -from validation import load_runner_file + +def load_runner_config(runner_config_path: str) -> dict: + """Load runner configuration for runtime model resolution.""" + try: + with open(runner_config_path) as runner_config_file: + runner_config = yaml.safe_load(runner_config_file) + except FileNotFoundError as error: + raise ValueError( + f"Runner config file '{runner_config_path}' does not exist." + ) from error + if not isinstance(runner_config, dict): + raise ValueError("Runner config must contain a mapping") + return runner_config def resolve_runner_model( runner_config: dict, runner_label: str, + framework: str, model_prefix: str, precision: str, current_model_path: Optional[str] = None, + fallback_model_path: Optional[str] = None, path_is_dir: Callable[[str], bool] = os.path.isdir, -) -> tuple[str, str]: - """Return the model path and srt-slurm alias for a runner/model pair.""" +) -> dict[str, str]: + """Return model environment values for a runner/framework/model tuple.""" models = runner_config.get("models", {}) - try: - model_config = models[runner_label][model_prefix][precision] - except KeyError as error: + runner_models = models.get(runner_label, {}) + model_config = None + for framework_key in (framework, "default"): + framework_models = runner_models.get(framework_key, {}) + precision_models = framework_models.get(model_prefix, {}) + model_config = ( + precision_models.get(precision) + or precision_models.get("default") + ) + if model_config is not None: + break + + if model_config is None and fallback_model_path is not None: + return {"MODEL_PATH": fallback_model_path} + if model_config is None: raise ValueError( - "Unsupported runner/model/precision combination: " - f"{runner_label}/{model_prefix}/{precision}" - ) from error + "Unsupported runner/framework/model/precision combination: " + f"{runner_label}/{framework}/{model_prefix}/{precision}" + ) configured_paths = model_config["model-paths"] if ( @@ -44,17 +69,24 @@ def resolve_runner_model( configured_paths[0], ) - return model_path, model_config["srt-slurm-model-prefix"] + environment = {"MODEL_PATH": model_path} + optional_exports = { + "SRT_SLURM_MODEL_PREFIX": "srt-slurm-model-prefix", + "SERVED_MODEL_NAME": "served-model-name", + "MODEL_NAME": "model-name", + } + for environment_name, config_name in optional_exports.items(): + value = model_config.get(config_name) + if value is not None: + environment[environment_name] = value + return environment -def shell_exports(model_path: str, srt_slurm_model_prefix: str) -> str: +def shell_exports(environment: dict[str, str]) -> str: """Render resolved values as shell-safe export statements.""" return "\n".join( - [ - f"export MODEL_PATH={shlex.quote(model_path)}", - "export SRT_SLURM_MODEL_PREFIX=" - f"{shlex.quote(srt_slurm_model_prefix)}", - ] + f"export {name}={shlex.quote(value)}" + for name, value in environment.items() ) @@ -67,6 +99,7 @@ def parse_args() -> argparse.Namespace: help="Path to runners.yaml", ) parser.add_argument("--runner-label", required=True) + parser.add_argument("--framework", required=True) parser.add_argument("--model-prefix", required=True) parser.add_argument("--precision", required=True) parser.add_argument( @@ -74,6 +107,10 @@ def parse_args() -> argparse.Namespace: default=os.environ.get("MODEL_PATH"), help="Existing MODEL_PATH override, when the mapping allows it", ) + parser.add_argument( + "--fallback-model-path", + help="MODEL_PATH to export when no configured mapping exists", + ) return parser.parse_args() @@ -81,19 +118,21 @@ def main() -> int: """Resolve and print shell exports for a runner model mapping.""" args = parse_args() try: - runner_config = load_runner_file(args.runner_config) - model_path, srt_slurm_model_prefix = resolve_runner_model( + runner_config = load_runner_config(args.runner_config) + environment = resolve_runner_model( runner_config, args.runner_label, + args.framework, args.model_prefix, args.precision, args.current_model_path, + args.fallback_model_path, ) except ValueError as error: print(f"Error: {error}", file=sys.stderr) return 1 - print(shell_exports(model_path, srt_slurm_model_prefix)) + print(shell_exports(environment)) return 0