diff --git a/configs/runners.yaml b/configs/runners.yaml index 851b821ba2..7cdd53ccb5 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -330,3 +330,317 @@ hardware: cluster:mi355x-amds: available-cpu-dram-mib: 3_095_781 gpus-per-node: 8 +models: + # Cluster-local checkpoints used by launchers instead of downloading the + # portable HuggingFace model IDs in the benchmark configs. When multiple + # paths are listed, the first existing directory wins and the first entry is + # the deterministic fallback. Framework-specific mappings take precedence + # over the default mapping. + cluster:b200-dgxc: + default: + dsr1: + fp4: + # Per-node /raid is not populated reliably across the cluster. + model-paths: + - /scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /lustre/fsw/models/dsr1-0528-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dsv4: + fp4: + model-paths: + - /lustre/fsw/models/deepseek-v4-pro + - /lustre/fsw/models/dsv4-pro + - /lustre/fsw/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-v4-pro + # Preserve the operator override supported by the previous launcher. + allow-model-path-override: true + qwen3.5: + bf16: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B + srt-slurm-model-prefix: qwen3.5 + fp8: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B-FP8 + srt-slurm-model-prefix: qwen3.5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4 + srt-slurm-model-prefix: qwen3.5-fp4 + glm5: + fp8: + model-paths: + - /lustre/fsw/models/GLM-5-FP8 + srt-slurm-model-prefix: glm5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/GLM-5-NVFP4 + srt-slurm-model-prefix: glm5-fp4 + kimik2.5: + int4: + model-paths: + - /lustre/fsw/models/Kimi-K2.5 + srt-slurm-model-prefix: kimik2.5 + fp4: + model-paths: + - /lustre/fsw/models/Kimi-K2.5-NVFP4 + srt-slurm-model-prefix: kimik2.5-fp4 + minimaxm2.5: + fp8: + model-paths: + - /lustre/fsw/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + fp4: + model-paths: + - /lustre/fsw/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + gptoss: + fp4: + model-paths: + - /lustre/fsw/models/gpt-oss-120b + srt-slurm-model-prefix: gptoss + minimaxm3: + fp8: + # Day-zero model staged in the sa-shared-writable gharunners tree. + model-paths: + - /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: minimax-m3-mxfp8 + fp4: + model-paths: + - /scratch/fsw/models/MiniMax-M3-NVFP4 + srt-slurm-model-prefix: minimax-m3-nvfp4 + cluster:b300-nv: + default: + dsr1: + fp4: + model-paths: + - /data/models/dsr1-fp4 + served-model-name: deepseek-r1-fp4 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /data/models/dsr1-fp8 + served-model-name: deepseek-r1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dynamo-vllm: + dsv4: + fp4: + model-paths: + - /data/models/dsv4-pro + - /data/models/deepseek-v4-pro + - /data/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-v4-pro + allow-model-path-override: true + minimaxm2.5: + fp4: + model-paths: + - /data/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + fp8: + model-paths: + - /data/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + minimaxm3: + fp4: + model-paths: + - /scratch/models/MiniMax-M3-NVFP4 + srt-slurm-model-prefix: nvidia/MiniMax-M3-NVFP4 + fp8: + model-paths: + - /data/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: MiniMaxAI/MiniMax-M3-MXFP8 + cluster:gb200-nv: + llmd-vllm: + dsv4: + fp4: + model-paths: + - /mnt/numa1/models/DeepSeek-V4-Pro + model-name: deepseek-ai/DeepSeek-V4-Pro + dynamo-sglang: + dsr1: + fp8: + model-paths: + - /mnt/lustre01/models/deepseek-r1-0528 + srt-slurm-model-prefix: dsr1-fp8 + fp4: + model-paths: + - /mnt/lustre01/models/deepseek-r1-0528-fp4-v2/ + srt-slurm-model-prefix: dsr1-fp4 + dsv4: + fp4: + model-paths: + - /mnt/lustre01/models/deepseek-v4-pro + srt-slurm-model-prefix: deepseek-v4-pro + glm5.1: + fp4: + model-paths: + - /mnt/lustre01/models/GLM-5.1-NVFP4 + srt-slurm-model-prefix: glm-5-fp4 + fp8: + model-paths: + - /mnt/lustre01/models/GLM-5.1-FP8 + srt-slurm-model-prefix: glm-5.1-fp8 + qwen3.5: + fp8: + model-paths: + - /mnt/lustre01/models/Qwen3.5-397B-A17B-FP8 + srt-slurm-model-prefix: qwen3.5-fp8 + dynamo-trt: + gptoss: + default: + model-paths: + - /mnt/lustre01/models/gpt-oss-120b + served-model-name: gpt-oss-120b + dsr1: + fp4: + model-paths: + - /mnt/numa1/models/DeepSeek-R1-0528-NVFP4-v2 + served-model-name: deepseek-r1-fp4 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /mnt/numa1/models/DeepSeek-R1-0528 + served-model-name: deepseek-r1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + kimik2.5: + fp4: + model-paths: + - /mnt/lustre01/models/kimi-k2.5-nvfp4 + served-model-name: kimi-k2.5-nvfp4 + srt-slurm-model-prefix: nvidia/Kimi-K2.5-NVFP4 + glm5: + fp4: + model-paths: + - /mnt/lustre01/slurm-shared/glm-model/GLM-5-NVFP4 + served-model-name: glm-5-nvfp4 + srt-slurm-model-prefix: nvidia/GLM-5-NVFP4 + dynamo-vllm: + kimik2.5: + fp4: + model-paths: + - /mnt/lustre01/models/kimi-k2.5-nvfp4 + srt-slurm-model-prefix: kimi-k2.5-nvfp4 + dsv4: + fp4: + # Compute-visible NVFP4 checkpoint; the lowercase sibling is FP8. + model-paths: + - /mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/ + srt-slurm-model-prefix: deepseek-v4-pro + minimaxm2.5: + fp4: + model-paths: + - /mnt/lustre01/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + fp8: + model-paths: + - /mnt/lustre01/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + minimaxm3: + fp8: + model-paths: + - /mnt/lustre01/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: minimax-m3-mxfp8 + cluster:gb300-nv: + default: + dsr1: + fp4: + model-paths: + - /scratch/models/DeepSeek-R1-0528-NVFP4-v2 + served-model-name: deepseek-r1-fp4 + srt-slurm-model-prefix: dsr1 + fp8: + model-paths: + - /scratch/models/DeepSeek-R1-0528 + served-model-name: deepseek-r1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dsv4: + fp4: + # Compute-node-local path; srtctl preflight must be skipped. + model-paths: + - /scratch/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-v4-pro + glm5.1: + fp4: + model-paths: + - /scratch/models/GLM-5.1-NVFP4 + srt-slurm-model-prefix: glm-5-fp4 + glm5: + fp4: + model-paths: + - /scratch/models/GLM-5-NVFP4 + srt-slurm-model-prefix: glm-5-fp4 + fp8: + model-paths: + - /scratch/models/GLM-5-FP8 + srt-slurm-model-prefix: glm-5-fp8 + minimaxm2.5: + fp4: + model-paths: + - /data/models/MiniMax-M2.5-NVFP4 + srt-slurm-model-prefix: minimax-m2.5-nvfp4 + fp8: + model-paths: + - /data/models/MiniMax-M2.5 + srt-slurm-model-prefix: minimax-m2.5-fp8 + minimaxm3: + fp8: + model-paths: + - /data/models/MiniMax-M3-MXFP8 + srt-slurm-model-prefix: minimax-m3-mxfp8 + kimik2.5: + fp4: + model-paths: + - /scratch/models/Kimi-K2.5-NVFP4 + srt-slurm-model-prefix: nvidia/Kimi-K2.5-NVFP4 + qwen3.5: + fp4: + model-paths: + - /scratch/models/Qwen3.5-397B-A17B-NVFP4 + srt-slurm-model-prefix: qwen3.5-fp4 + fp8: + model-paths: + - /scratch/models/Qwen3.5-397B-A17B-FP8 + srt-slurm-model-prefix: qwen3.5-fp8 + dynamo-trt: + dsv4: + fp4: + model-paths: + - /scratch/models/DeepSeek-V4-Pro + srt-slurm-model-prefix: deepseek-ai/DeepSeek-V4-Pro + glm5: + fp4: + model-paths: + - /scratch/models/GLM-5-NVFP4 + served-model-name: glm-5-nvfp4 + srt-slurm-model-prefix: nvidia/GLM-5-NVFP4 + cluster:h100-dgxc: + dynamo-sglang: + dsr1: + fp8: + model-paths: + - /mnt/nfs/lustre/models/dsr1-fp8 + srt-slurm-model-prefix: dsr1-fp8 + dynamo-trt: + dsr1: + fp8: + model-paths: + - /mnt/nfs/lustre/models/dsr1-fp8 + served-model-name: DeepSeek-R1-0528 + srt-slurm-model-prefix: DeepSeek-R1-0528 + cluster:h200-dgxc: + dynamo-sglang: + dsr1: + fp8: + model-paths: + - /models/DeepSeek-R1-0528 + srt-slurm-model-prefix: dsr1-fp8 + dynamo-trt: + dsr1: + fp8: + model-paths: + - /models/DeepSeek-R1-0528 + served-model-name: DeepSeek-R1-0528 + srt-slurm-model-prefix: DeepSeek-R1-0528 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a276644575..d86fe76d29 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -6,78 +6,9 @@ SLURM_ACCOUNT="benchmark" set -x -# MODEL_PATH: Override with pre-downloaded paths on the shared Lustre tree. -# Bench scripts and srt-slurm yaml configs specify HuggingFace model IDs for -# portability, but we resolve to /lustre/fsw/models/* here to avoid repeated -# downloading on every dgxc node. Runs for both single-node and multinode -# launches. -# NOTE: per-node /raid/models/* would be faster but is only populated on a -# subset of dgxc nodes today, so we use Lustre for reliability. -if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2" - export SRT_SLURM_MODEL_PREFIX="dsr1" -elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/dsr1-0528-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - SELECTED_MODEL_PATH="" - if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then - SELECTED_MODEL_PATH="$MODEL_PATH" - else - for candidate in /lustre/fsw/models/deepseek-v4-pro /lustre/fsw/models/dsv4-pro /lustre/fsw/models/DeepSeek-V4-Pro; do - if [[ -d "$candidate" ]]; then - SELECTED_MODEL_PATH="$candidate" - break - fi - done - fi - export MODEL_PATH="${SELECTED_MODEL_PATH:-/lustre/fsw/models/deepseek-v4-pro}" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "bf16" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B" - export SRT_SLURM_MODEL_PREFIX="qwen3.5" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-FP8" - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4" - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/GLM-5-FP8" - export SRT_SLURM_MODEL_PREFIX="glm5-fp8" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/GLM-5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="glm5-fp4" -elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5" - export SRT_SLURM_MODEL_PREFIX="kimik2.5" -elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" -elif [[ $MODEL_PREFIX == "gptoss" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/gpt-oss-120b" - export SRT_SLURM_MODEL_PREFIX="gptoss" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - # Day-zero: MiniMax-M3-MXFP8 is not in the SRE-staged /lustre/fsw/models - # tree (root-owned); it lives in the sa-shared-writable gharunners tree. - export MODEL_PATH="/lustre/fsw/gharunners/models/MiniMax-M3-MXFP8" - export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then - # NVFP4 checkpoint, pre-staged on the b200-dgxc scratch tree. - export MODEL_PATH="/scratch/fsw/models/MiniMax-M3-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" -else - echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION" - echo "Available models under /lustre/fsw/models:" - ls -la /lustre/fsw/models - exit 1 -fi +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" +resolve_runner_model_config "cluster:b200-dgxc" || exit 1 export AIPERF_MMAP_CACHE_HOST_PATH="/lustre/fsw/gharunners/aiperf-cache" diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 098e984be5..6b0e30215c 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -8,6 +8,9 @@ MINIMAX_M3_SLURM_EXCLUDED_NODELIST="${MINIMAX_M3_SLURM_EXCLUDED_NODELIST-b300-01 set -x +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + if [[ "$IS_MULTINODE" == "true" ]]; then # Validate framework @@ -16,47 +19,7 @@ if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" && $FRAMEWORK exit 1 fi -# MODEL_PATH: Override with pre-downloaded paths on B300 runner -# The yaml files specify HuggingFace model IDs for portability, but we use -# local paths to avoid repeated downloading on the shared B300 cluster. -if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/data/models/dsr1-fp4" - export SERVED_MODEL_NAME="deepseek-r1-fp4" - export SRT_SLURM_MODEL_PREFIX="dsr1" -elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/data/models/dsr1-fp8" - export SERVED_MODEL_NAME="deepseek-r1-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then - SELECTED_MODEL_PATH="" - if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then - SELECTED_MODEL_PATH="$MODEL_PATH" - else - for candidate in /data/models/dsv4-pro /data/models/deepseek-v4-pro /data/models/DeepSeek-V4-Pro; do - if [[ -d "$candidate" ]]; then - SELECTED_MODEL_PATH="$candidate" - break - fi - done - fi - export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/data/models/MiniMax-M2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/data/models/MiniMax-M2.5" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/scratch/models/MiniMax-M3-NVFP4" - export SRT_SLURM_MODEL_PREFIX="nvidia/MiniMax-M3-NVFP4" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dynamo-vllm" ]]; then - export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" - export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" -else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" - exit 1 -fi +resolve_runner_model_config "cluster:b300-nv" || exit 1 echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 426cd2dd87..da64129f87 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -5,19 +5,15 @@ set -x source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" export SLURM_PARTITION="batch" export SLURM_ACCOUNT="benchmark" SQUASH_DIR="/mnt/lustre01/users-public/sa-shared" if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then - if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then - export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" - export MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro" - else - echo "Unsupported MODEL_PREFIX/PRECISION for llmd-vllm on GB200: $MODEL_PREFIX/$PRECISION" >&2 - exit 1 - fi + resolve_runner_model_config "cluster:gb200-nv" || exit 1 SQUASH_FILE="${SQUASH_DIR}/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" @@ -87,96 +83,15 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then exit 0 fi -# MODEL_PATH: Override with pre-downloaded paths on GB200 runner -# The yaml files specify HuggingFace model IDs for portability, but we use -# local paths to avoid repeated downloading on the shared GB200 cluster. if [[ $FRAMEWORK == "dynamo-sglang" ]]; then export CONFIG_DIR="/mnt/lustre01/artifacts/sglang-configs/1k1k" - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528-fp4-v2/" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp4" - elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # Lustre-resident weights staged on the GB200 external cluster. - # SRT_SLURM_MODEL_PREFIX matches the model.path alias in our - # DSV4 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/deepseek-v4-pro" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" - elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-NVFP4" - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" - elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-FP8" - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" - elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-NVFP4" - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" - elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5.1-fp8") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-FP8" - export SRT_SLURM_MODEL_PREFIX="glm-5.1-fp8" - else - export MODEL_PATH=$MODEL - fi -elif [[ $FRAMEWORK == "dynamo-trt" ]]; then - if [[ $MODEL_PREFIX == "gptoss" ]]; then - export MODEL_PATH="/mnt/lustre01/models/gpt-oss-120b" - export SERVED_MODEL_NAME="gpt-oss-120b" - elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/numa1/models/DeepSeek-R1-0528-NVFP4-v2" - export SERVED_MODEL_NAME="deepseek-r1-fp4" - export SRT_SLURM_MODEL_PREFIX="dsr1" - elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/numa1/models/DeepSeek-R1-0528" - export SERVED_MODEL_NAME="deepseek-r1-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" - export SERVED_MODEL_NAME="kimi-k2.5-nvfp4" - export SRT_SLURM_MODEL_PREFIX="nvidia/Kimi-K2.5-NVFP4" - elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias - # ("nvidia/GLM-5-NVFP4") in the upstream GLM5 trtllm_dynamo recipes. - export MODEL_PATH="/mnt/lustre01/slurm-shared/glm-model/GLM-5-NVFP4" - export SERVED_MODEL_NAME="glm-5-nvfp4" - export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5-NVFP4" - else - echo "Unsupported model prefix: $MODEL_PREFIX. Supported prefixes are: gptoss, dsr1, kimik2.5, or glm5" - exit 1 - fi -elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then - if [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" - export SRT_SLURM_MODEL_PREFIX="kimi-k2.5-nvfp4" - elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # The FP4 checkpoint is staged on compute-visible Lustre. The former - # /mnt/numa1 path is no longer present on watchtower compute nodes; - # the lowercase Lustre sibling is the FP8 checkpoint, so keep the - # NVFP4 path explicit here. - export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" - elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" - elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5" - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" - elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-MXFP8" - export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" - else - echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp8" - exit 1 - fi + resolve_runner_model_config "cluster:gb200-nv" \ + --fallback-model-path "$MODEL" || exit 1 +elif [[ $FRAMEWORK == "dynamo-trt" || $FRAMEWORK == "dynamo-vllm" ]]; then + resolve_runner_model_config "cluster:gb200-nv" || exit 1 else - export MODEL_PATH=$MODEL + resolve_runner_model_config "cluster:gb200-nv" \ + --fallback-model-path "$MODEL" || exit 1 fi NGINX_IMAGE="nginx:1.27.4" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index e5a8d059b2..34e87f683f 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -4,6 +4,9 @@ set -exo pipefail +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + export SLURM_PARTITION="batch_1" export SLURM_ACCOUNT="benchmark" export ENROOT_ROOTFS_WRITABLE=1 @@ -31,69 +34,7 @@ mkdir -p "$HF_HUB_CACHE_HOST_PATH" export DYNAMO_WHEELS_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/dynamo-wheels" mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH" -export MODEL_PATH=$MODEL - -if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - export SERVED_MODEL_NAME="deepseek-r1-fp4" - export MODEL_PATH=/scratch/models/DeepSeek-R1-0528-NVFP4-v2 - export SRT_SLURM_MODEL_PREFIX="dsr1" -elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export SERVED_MODEL_NAME="deepseek-r1-fp8" - export MODEL_PATH=/scratch/models/DeepSeek-R1-0528 - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # Use the node-local /scratch SSD for the 806 GB DSv4-Pro - # checkpoint. Faster than the Vast NFS path, but this dir only - # exists on compute nodes — the GHA runner pod's view does NOT - # have /scratch/models, so srtctl preflight (which stats the path - # from the runner pod) may fail with "Model alias resolved to - # /scratch/models/DeepSeek-V4-Pro, but that path is unavailable." - # If that happens, the next step is either to (a) patch srt-slurm - # to add a skip_model_preflight recipe field, or (b) stub a - # symlink on the runner pod that points at the NFS copy. - export MODEL_PATH=/scratch/models/DeepSeek-V4-Pro - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-trt" ]]; then - export SERVED_MODEL_NAME="glm-5-nvfp4" - export MODEL_PATH=/scratch/models/GLM-5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5-NVFP4" -elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") - # in our GLM-5.1 sglang recipes. - export MODEL_PATH=/scratch/models/GLM-5.1-NVFP4 - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH=/scratch/models/GLM-5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" -elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH=/scratch/models/GLM-5-FP8 - export SRT_SLURM_MODEL_PREFIX="glm-5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH=/data/models/MiniMax-M2.5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" -elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH=/data/models/MiniMax-M2.5 - export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" -elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - export MODEL_PATH=/data/models/MiniMax-M3-MXFP8 - export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" -elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH=/scratch/models/Kimi-K2.5-NVFP4 - export SRT_SLURM_MODEL_PREFIX="nvidia/Kimi-K2.5-NVFP4" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our - # Qwen3.5 sglang recipes (qwen3.5-fp4). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then - # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our - # Qwen3.5 sglang recipes (qwen3.5-fp8). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-FP8 - export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" -else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" - exit 1 -fi +resolve_runner_model_config "cluster:gb300-nv" || exit 1 NGINX_IMAGE="nginx:1.27.4" @@ -223,9 +164,6 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.5" && $PRECISION mkdir -p recipes/vllm/kimi-k2.5-fp4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4" recipes/vllm/kimi-k2.5-fp4 elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 dynamo-trt recipes use the HuggingFace model ID as model.path, - # so override SRT_SLURM_MODEL_PREFIX to match the recipe's model path key. - SRT_SLURM_MODEL_PREFIX="deepseek-ai/DeepSeek-V4-Pro" git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 1334c95542..c9d948cfba 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -11,32 +11,16 @@ SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') set -x +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + if [[ "$IS_MULTINODE" == "true" ]]; then - # MODEL_PATH: Override with pre-downloaded paths on H100 runner - # The yaml files specify HuggingFace model IDs for portability, but we use - # local paths to avoid repeated downloading on the shared H100 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/nfs/lustre/models/dsr1-fp8" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - else - echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - elif [[ $FRAMEWORK == "dynamo-trt" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/nfs/lustre/models/dsr1-fp8" - export SERVED_MODEL_NAME="DeepSeek-R1-0528" - export SRT_SLURM_MODEL_PREFIX="DeepSeek-R1-0528" - else - echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - else + if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" ]]; then echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" exit 1 fi + resolve_runner_model_config "cluster:h100-dgxc" || exit 1 echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 00c6cc4977..3bf0cc5c66 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -7,32 +7,16 @@ SLURM_ACCOUNT="sa-shared" set -x +# shellcheck source=runners/runner_model_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/runner_model_utils.sh" + if [[ "$IS_MULTINODE" == "true" ]]; then - # MODEL_PATH: Override with pre-downloaded paths on H200 runner - # The yaml files specify HuggingFace model IDs for portability, but we use - # local paths to avoid repeated downloading on the shared H200 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/models/DeepSeek-R1-0528" - export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" - else - echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - elif [[ $FRAMEWORK == "dynamo-trt" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/models/DeepSeek-R1-0528" - export SERVED_MODEL_NAME="DeepSeek-R1-0528" - export SRT_SLURM_MODEL_PREFIX="DeepSeek-R1-0528" - else - echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" - exit 1 - fi - else + if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" ]]; then echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" exit 1 fi + resolve_runner_model_config "cluster:h200-dgxc" || exit 1 echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" diff --git a/runners/runner_model_utils.sh b/runners/runner_model_utils.sh new file mode 100644 index 0000000000..a31cdccd4e --- /dev/null +++ b/runners/runner_model_utils.sh @@ -0,0 +1,23 @@ +#!/usr/bin/bash + +# Resolve a launcher's cluster-local model environment from configs/runners.yaml. +resolve_runner_model_config() { + local runner_label="$1" + shift + + local runner_utils_dir repo_root resolved_model_env + runner_utils_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + repo_root="$(cd "$runner_utils_dir/.." && pwd)" + + if ! resolved_model_env=$(python3 "$repo_root/utils/resolve_runner_model.py" \ + --runner-config "$repo_root/configs/runners.yaml" \ + --runner-label "$runner_label" \ + --framework "$FRAMEWORK" \ + --model-prefix "$MODEL_PREFIX" \ + --precision "$PRECISION" \ + "$@"); then + return 1 + fi + + eval "$resolved_model_env" +} diff --git a/utils/matrix_logic/test_resolve_runner_model.py b/utils/matrix_logic/test_resolve_runner_model.py new file mode 100644 index 0000000000..bcaad81800 --- /dev/null +++ b/utils/matrix_logic/test_resolve_runner_model.py @@ -0,0 +1,285 @@ +"""Tests for cluster-local model path resolution.""" + +from pathlib import Path + +import pytest + +from utils.matrix_logic.validation import load_config_files, load_runner_file +from utils.resolve_runner_model import resolve_runner_model, shell_exports + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def runner_config(model_paths, allow_override=False): + """Build a minimal runner model configuration.""" + return { + "models": { + "cluster:b200-dgxc": { + "default": { + "dsv4": { + "fp4": { + "model-paths": model_paths, + "srt-slurm-model-prefix": "deepseek-v4-pro", + "allow-model-path-override": allow_override, + } + } + } + } + } + } + + +def test_resolve_uses_first_existing_configured_path(): + """The first existing configured path wins.""" + config = runner_config(["/models/preferred", "/models/fallback"]) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-vllm", + "dsv4", + "fp4", + path_is_dir=lambda path: path == "/models/fallback", + ) + + assert result == { + "MODEL_PATH": "/models/fallback", + "SRT_SLURM_MODEL_PREFIX": "deepseek-v4-pro", + } + + +def test_resolve_falls_back_to_first_path_when_none_exist(): + """The first path remains the deterministic fallback.""" + config = runner_config(["/models/preferred", "/models/fallback"]) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-vllm", + "dsv4", + "fp4", + path_is_dir=lambda _path: False, + ) + + assert result["MODEL_PATH"] == "/models/preferred" + + +def test_resolve_honors_allowed_existing_override(): + """An allowed, existing MODEL_PATH override takes precedence.""" + config = runner_config(["/models/preferred"], allow_override=True) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-vllm", + "dsv4", + "fp4", + current_model_path="/models/operator-override", + path_is_dir=lambda path: path == "/models/operator-override", + ) + + assert result["MODEL_PATH"] == "/models/operator-override" + + +def test_resolve_prefers_framework_specific_mapping(): + """An exact framework mapping overrides the runner default.""" + config = runner_config(["/models/default"]) + config["models"]["cluster:b200-dgxc"]["dynamo-trt"] = { + "dsv4": { + "fp4": { + "model-paths": ["/models/trt"], + "srt-slurm-model-prefix": "deepseek-ai/DeepSeek-V4-Pro", + "served-model-name": "deepseek-v4-pro", + } + } + } + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-trt", + "dsv4", + "fp4", + path_is_dir=lambda _path: False, + ) + + assert result == { + "MODEL_PATH": "/models/trt", + "SRT_SLURM_MODEL_PREFIX": "deepseek-ai/DeepSeek-V4-Pro", + "SERVED_MODEL_NAME": "deepseek-v4-pro", + } + + +def test_resolve_supports_default_precision_mapping(): + """A default precision supports model mappings that ignore precision.""" + config = runner_config(["/models/default"]) + config["models"]["cluster:b200-dgxc"]["dynamo-trt"] = { + "gptoss": { + "default": { + "model-paths": ["/models/gpt-oss"], + "served-model-name": "gpt-oss-120b", + } + } + } + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-trt", + "gptoss", + "mxfp4", + path_is_dir=lambda _path: False, + ) + + assert result == { + "MODEL_PATH": "/models/gpt-oss", + "SERVED_MODEL_NAME": "gpt-oss-120b", + } + + +def test_resolve_supports_explicit_missing_mapping_fallback(): + """Callers may preserve a portable MODEL fallback for unmapped models.""" + config = runner_config(["/models/default"]) + + result = resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-sglang", + "qwen", + "bf16", + fallback_model_path="Qwen/Qwen3", + path_is_dir=lambda _path: False, + ) + + assert result == {"MODEL_PATH": "Qwen/Qwen3"} + + +def test_resolve_rejects_unsupported_combination(): + """Missing model combinations fail with the full lookup key.""" + config = runner_config(["/models/preferred"]) + + with pytest.raises(ValueError) as exc_info: + resolve_runner_model( + config, + "cluster:b200-dgxc", + "dynamo-vllm", + "dsr1", + "fp8", + path_is_dir=lambda _path: False, + ) + + assert ( + "cluster:b200-dgxc/dynamo-vllm/dsr1/fp8" + in str(exc_info.value) + ) + + +def test_shell_exports_quote_values(): + """Shell output quotes values instead of interpolating them.""" + exports = shell_exports( + { + "MODEL_PATH": "/models/path with spaces", + "SRT_SLURM_MODEL_PREFIX": "model;alias", + } + ) + + assert exports == ( + "export MODEL_PATH='/models/path with spaces'\n" + "export SRT_SLURM_MODEL_PREFIX='model;alias'" + ) + + +def test_resolve_every_configured_runner_model_mapping(): + """Every runners.yaml model entry is reachable through the resolver.""" + config = load_runner_file(str(REPO_ROOT / "configs" / "runners.yaml")) + export_names = { + "srt-slurm-model-prefix": "SRT_SLURM_MODEL_PREFIX", + "served-model-name": "SERVED_MODEL_NAME", + "model-name": "MODEL_NAME", + } + + for runner_label, framework_mappings in config["models"].items(): + for framework_key, model_mappings in framework_mappings.items(): + framework = ( + "unconfigured-framework" + if framework_key == "default" + else framework_key + ) + for model_prefix, precision_mappings in model_mappings.items(): + for precision_key, model_config in precision_mappings.items(): + precision = ( + "unconfigured-precision" + if precision_key == "default" + else precision_key + ) + result = resolve_runner_model( + config, + runner_label, + framework, + model_prefix, + precision, + path_is_dir=lambda _path: False, + ) + + assert result["MODEL_PATH"] == model_config["model-paths"][0] + for config_name, export_name in export_names.items(): + if config_name in model_config: + assert result[export_name] == model_config[config_name] + + +def test_every_nvidia_multinode_runner_model_resolves(): + """Every multinode config routed to a migrated launcher is supported.""" + runner_config = load_runner_file( + str(REPO_ROOT / "configs" / "runners.yaml") + ) + master_config = load_config_files( + [str(REPO_ROOT / "configs" / "nvidia-master.yaml")] + ) + cluster_by_runner = { + "b200": "cluster:b200-dgxc", + "b200-dsv4": "cluster:b200-dgxc", + "b200-multinode": "cluster:b200-dgxc", + "cluster:b200-dgxc": "cluster:b200-dgxc", + "b300": "cluster:b300-nv", + "b300-p1": "cluster:b300-nv", + "cluster:b300-nv": "cluster:b300-nv", + "gb200": "cluster:gb200-nv", + "cluster:gb200-nv": "cluster:gb200-nv", + "gb300": "cluster:gb300-nv", + "gb300-nv": "cluster:gb300-nv", + "cluster:gb300-nv": "cluster:gb300-nv", + "h100-multinode": "cluster:h100-dgxc", + "cluster:h100-dgxc": "cluster:h100-dgxc", + "h200-multinode": "cluster:h200-dgxc", + "cluster:h200-dgxc": "cluster:h200-dgxc", + } + strict_gb200_frameworks = {"llmd-vllm", "dynamo-trt", "dynamo-vllm"} + unsupported = [] + + for config_name, benchmark in master_config.items(): + if not benchmark.get("multinode", False): + continue + runner_label = cluster_by_runner.get(benchmark["runner"]) + if runner_label is None: + continue + fallback_model_path = None + if ( + runner_label == "cluster:gb200-nv" + and benchmark["framework"] not in strict_gb200_frameworks + ): + fallback_model_path = benchmark["model"] + + try: + resolve_runner_model( + runner_config, + runner_label, + benchmark["framework"], + benchmark["model-prefix"], + benchmark["precision"], + fallback_model_path=fallback_model_path, + path_is_dir=lambda _path: False, + ) + except ValueError as error: + unsupported.append(f"{config_name}: {error}") + + assert not unsupported, "\n".join(unsupported) diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 3d25d03eef..32d24a617c 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -1359,6 +1359,92 @@ def test_hardware_gpus_per_node_must_be_positive(self): validate_runner_config(config) assert "gpus-per-node" in str(exc_info.value) + def test_valid_runner_model_mapping(self): + config = { + "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "dynamo-vllm": { + "dsv4": { + "fp4": { + "model-paths": [ + "/lustre/fsw/models/deepseek-v4-pro", + "/lustre/fsw/models/dsv4-pro", + ], + "srt-slurm-model-prefix": "deepseek-v4-pro", + "allow-model-path-override": True, + } + } + } + } + }, + } + + assert validate_runner_config(config) == config + + def test_runner_model_mapping_requires_known_label(self): + config = { + "labels": {"b200": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "default": { + "dsr1": { + "fp8": { + "model-paths": ["/lustre/fsw/models/dsr1"], + "srt-slurm-model-prefix": "dsr1-fp8", + } + } + } + } + }, + } + + with pytest.raises(ValueError, match="unknown labels"): + validate_runner_config(config) + + def test_runner_model_paths_must_be_absolute(self): + config = { + "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "default": { + "dsr1": { + "fp8": { + "model-paths": ["relative/model/path"], + "srt-slurm-model-prefix": "dsr1-fp8", + } + } + } + } + }, + } + + with pytest.raises(ValueError, match="must be absolute"): + validate_runner_config(config) + + def test_runner_model_paths_must_be_unique(self): + config = { + "labels": {"cluster:b200-dgxc": ["b200-dgxc_0"]}, + "models": { + "cluster:b200-dgxc": { + "default": { + "dsr1": { + "fp8": { + "model-paths": [ + "/lustre/fsw/models/dsr1", + "/lustre/fsw/models/dsr1", + ], + "srt-slurm-model-prefix": "dsr1-fp8", + } + } + } + } + }, + } + + with pytest.raises(ValueError, match="must not contain duplicates"): + validate_runner_config(config) + # ============================================================================= # Test changelog entry validation diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 9f19572f08..d9e784f55f 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -831,12 +831,65 @@ class RunnerHardwareConfig(BaseModel): ) +class RunnerModelConfig(BaseModel): + """Cluster-local model paths and aliases used by runner launchers.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + model_paths: List[str] = Field(alias='model-paths', min_length=1) + srt_slurm_model_prefix: Optional[str] = Field( + alias='srt-slurm-model-prefix', default=None, min_length=1 + ) + served_model_name: Optional[str] = Field( + alias='served-model-name', default=None, min_length=1 + ) + model_name: Optional[str] = Field( + alias='model-name', default=None, min_length=1 + ) + allow_model_path_override: bool = Field( + alias='allow-model-path-override', default=False + ) + + @field_validator('model_paths') + @classmethod + def validate_model_paths(cls, model_paths: List[str]) -> List[str]: + """Require unique absolute paths that are safe to export to a shell.""" + if len(model_paths) != len(set(model_paths)): + raise ValueError("'model-paths' must not contain duplicates") + for model_path in model_paths: + if not model_path.startswith('/'): + raise ValueError( + f"Runner model path must be absolute: {model_path}" + ) + if '\n' in model_path or '\t' in model_path: + raise ValueError( + "Runner model paths must not contain tabs or newlines" + ) + return model_paths + + class RunnerConfig(BaseModel): """Top-level runner configuration file.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) labels: Dict[str, List[str]] hardware: Dict[str, RunnerHardwareConfig] = Field(default_factory=dict) + models: Dict[ + str, + Dict[str, Dict[str, Dict[str, RunnerModelConfig]]] + ] = Field( + default_factory=dict + ) + + @model_validator(mode='after') + def validate_model_runner_labels(self): + """Require model mappings to reference a declared runner label.""" + unknown_labels = sorted(set(self.models) - set(self.labels)) + if unknown_labels: + raise ValueError( + "Runner model mappings reference unknown labels: " + + ", ".join(unknown_labels) + ) + return self def validate_runner_config(runner_configs: dict) -> dict: diff --git a/utils/resolve_runner_model.py b/utils/resolve_runner_model.py new file mode 100644 index 0000000000..9dcc607e7f --- /dev/null +++ b/utils/resolve_runner_model.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Resolve cluster-local model paths from configs/runners.yaml.""" + +import argparse +import os +import shlex +import sys +from typing import Callable, Optional + +import yaml + + +def load_runner_config(runner_config_path: str) -> dict: + """Load runner configuration for runtime model resolution.""" + try: + with open(runner_config_path) as runner_config_file: + runner_config = yaml.safe_load(runner_config_file) + except FileNotFoundError as error: + raise ValueError( + f"Runner config file '{runner_config_path}' does not exist." + ) from error + if not isinstance(runner_config, dict): + raise ValueError("Runner config must contain a mapping") + return runner_config + + +def resolve_runner_model( + runner_config: dict, + runner_label: str, + framework: str, + model_prefix: str, + precision: str, + current_model_path: Optional[str] = None, + fallback_model_path: Optional[str] = None, + path_is_dir: Callable[[str], bool] = os.path.isdir, +) -> dict[str, str]: + """Return model environment values for a runner/framework/model tuple.""" + models = runner_config.get("models", {}) + runner_models = models.get(runner_label, {}) + model_config = None + for framework_key in (framework, "default"): + framework_models = runner_models.get(framework_key, {}) + precision_models = framework_models.get(model_prefix, {}) + model_config = ( + precision_models.get(precision) + or precision_models.get("default") + ) + if model_config is not None: + break + + if model_config is None and fallback_model_path is not None: + return {"MODEL_PATH": fallback_model_path} + if model_config is None: + raise ValueError( + "Unsupported runner/framework/model/precision combination: " + f"{runner_label}/{framework}/{model_prefix}/{precision}" + ) + + configured_paths = model_config["model-paths"] + if ( + model_config.get("allow-model-path-override", False) + and current_model_path + and path_is_dir(current_model_path) + ): + model_path = current_model_path + else: + model_path = next( + (path for path in configured_paths if path_is_dir(path)), + configured_paths[0], + ) + + environment = {"MODEL_PATH": model_path} + optional_exports = { + "SRT_SLURM_MODEL_PREFIX": "srt-slurm-model-prefix", + "SERVED_MODEL_NAME": "served-model-name", + "MODEL_NAME": "model-name", + } + for environment_name, config_name in optional_exports.items(): + value = model_config.get(config_name) + if value is not None: + environment[environment_name] = value + return environment + + +def shell_exports(environment: dict[str, str]) -> str: + """Render resolved values as shell-safe export statements.""" + return "\n".join( + f"export {name}={shlex.quote(value)}" + for name, value in environment.items() + ) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--runner-config", + default="configs/runners.yaml", + help="Path to runners.yaml", + ) + parser.add_argument("--runner-label", required=True) + parser.add_argument("--framework", required=True) + parser.add_argument("--model-prefix", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument( + "--current-model-path", + default=os.environ.get("MODEL_PATH"), + help="Existing MODEL_PATH override, when the mapping allows it", + ) + parser.add_argument( + "--fallback-model-path", + help="MODEL_PATH to export when no configured mapping exists", + ) + return parser.parse_args() + + +def main() -> int: + """Resolve and print shell exports for a runner model mapping.""" + args = parse_args() + try: + runner_config = load_runner_config(args.runner_config) + environment = resolve_runner_model( + runner_config, + args.runner_label, + args.framework, + args.model_prefix, + args.precision, + args.current_model_path, + args.fallback_model_path, + ) + except ValueError as error: + print(f"Error: {error}", file=sys.stderr) + return 1 + + print(shell_exports(environment)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())