From 2567e886ef5de41240704d77011cbaa23c76f872 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 16:25:31 -0400 Subject: [PATCH 1/6] feat: add MiniMax M3 MXFP8 GB200 Dynamo-vLLM EAGLE3 MTP recipes and fix non-MTP comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add minimaxm3-fp8-gb200-dynamo-vllm-mtp: two EAGLE3 SLURM recipes (1p2d-dep4-tep8 conc 16/64/128, 2p1d-dep4-dep16 conc 512/1024) under minimax-m3-gb200-fp8/8k1k/mtp/; launcher already cp-rT's the whole minimax-m3-gb200-fp8 tree so no runner changes needed - Fix minimaxm3-fp8-gb200-dynamo-vllm: correct 4 wrong comments (Xp2d DEP8 -> Xp1d DEP16), remove phantom Marlin entry from header, bump router version 1.3.0.dev20260614 -> 1.3.0.dev20260710 中文：新增 MiniMax M3 MXFP8 GB200 Dynamo-vLLM EAGLE3 投机解码配方并修正 非 MTP 配置注释 - 新增 minimaxm3-fp8-gb200-dynamo-vllm-mtp：两个 EAGLE3 SLURM 配方 （1p2d-dep4-tep8 并发 16/64/128，2p1d-dep4-dep16 并发 512/1024）， 位于 minimax-m3-gb200-fp8/8k1k/mtp/；启动器已通过 cp-rT 复制整个 minimax-m3-gb200-fp8 目录，无需修改 runner 脚本 - 修正 minimaxm3-fp8-gb200-dynamo-vllm：修正 4 处错误注释 （Xp2d DEP8 -> Xp1d DEP16），移除头注释中不存在的 Marlin 条目， 将 router 版本从 1.3.0.dev20260614 升级至 1.3.0.dev20260710 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 104 +++++++++++++++++ .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 106 ++++++++++++++++++ configs/nvidia-master.yaml | 59 +++++++++- perf-changelog.yaml | 7 ++ 4 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..3cc83e86ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -0,0 +1,104 @@ +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16x64x128" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml new file mode 100644 index 0000000000..1546180747 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..d974d3fb62 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8060,7 +8060,7 @@ qwen3.5-fp4-b200-trt: - { tp: 8, ep: 8, dp-attn: true, conc-list: [256, 512, 1024] } # MiniMax-M3 GB200 disagg sweep — adapted from NV B300 PR #1863. -# All prefill DEP4 (TP1 DP4 EP, 4 GPU/worker). Decode: Marlin, TEP8, DEP8, TEP4. +# All prefill DEP4 (TP1 DP4 EP, 4 GPU/worker). Decode: TEP8, DEP16, TEP4. # 4 GPU/node (GB200 NVL72). FLASHINFER attention with FP8 KV cache. minimaxm3-fp8-gb200-dynamo-vllm: image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 @@ -8069,7 +8069,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: runner: gb200 precision: fp8 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260614" } + router: { name: dynamo-router, version: "1.3.0.dev20260710" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8078,7 +8078,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: - isl: 8192 osl: 1024 search-space: - # 1p2d DEP4+DEP8, 5n: conc 512 + # 1p1d DEP4+DEP16, 5n: conc 512 - conc-list: [512] prefill: num-worker: 1 @@ -8123,7 +8123,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 8 dp-attn: false - # 2p2d DEP4+DEP8, 6n: conc 512,1024 + # 2p1d DEP4+DEP16, 6n: conc 512,1024 - conc-list: [512, 1024] prefill: num-worker: 2 @@ -8138,7 +8138,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true - # 3p2d DEP4+DEP8, 7n: conc 4096 + # 3p1d DEP4+DEP16, 7n: conc 4096 - conc-list: [4096] prefill: num-worker: 3 @@ -8153,7 +8153,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true - # 5p2d DEP4+DEP8, 9n: conc 4096 + # 5p1d DEP4+DEP16, 9n: conc 4096 - conc-list: [4096] prefill: num-worker: 5 @@ -8168,6 +8168,53 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true +minimaxm3-fp8-gb200-dynamo-vllm-mtp: + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: gb200 + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260710" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1p2d DEP4+TEP8, 5n: conc 16,64,128 MTP + - spec-decoding: "mtp" + conc-list: [16, 64, 128] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false + # 2p1d DEP4+DEP16, 6n: conc 512,1024 MTP + - spec-decoding: "mtp" + conc-list: [512, 1024] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..c5fead94a4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,10 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - minimaxm3-fp8-gb200-dynamo-vllm-mtp + description: + - "Add MiniMax M3 MXFP8 GB200 Dynamo-vLLM disaggregated EAGLE3 MTP recipes" + - "Fix comment errors in minimaxm3-fp8-gb200-dynamo-vllm (Xp2d DEP8 -> Xp1d DEP16, remove phantom Marlin entry, bump router to 1.3.0.dev20260710)" + pr-link: TBD From 08509712f93bc8d0e801018e8f36dede1b7670cf Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 16:26:16 -0400 Subject: [PATCH 2/6] chore: fill perf-changelog pr-link for minimaxm3-fp8-gb200-dynamo-vllm-mtp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：补全 minimaxm3-fp8-gb200-dynamo-vllm-mtp 的 perf-changelog pr-link Co-Authored-By: Claude Sonnet 4.6 (1M context) --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5fead94a4..d3544cb162 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5361,4 +5361,4 @@ description: - "Add MiniMax M3 MXFP8 GB200 Dynamo-vLLM disaggregated EAGLE3 MTP recipes" - "Fix comment errors in minimaxm3-fp8-gb200-dynamo-vllm (Xp2d DEP8 -> Xp1d DEP16, remove phantom Marlin entry, bump router to 1.3.0.dev20260710)" - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2432 From c1ef64f760e9838af03e5a326ffd04117dea7480 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 18:39:29 -0400 Subject: [PATCH 3/6] fix: change GB200 fp8 MTP prefill from TP1 DP4 to TP2 DP2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both EAGLE3 MTP recipe YAMLs and master-config tp/ep updated. 中文：将 GB200 fp8 MTP 预填充并行从 TP1 DP4 改为 TP2 DP2， 同步更新两个 EAGLE3 MTP 配方 YAML 及主配置 tp/ep 字段 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 4 ++-- .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 4 ++-- configs/nvidia-master.yaml | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml index 3cc83e86ae..7498395b18 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 + tensor-parallel-size: 2 + data-parallel-size: 2 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml index 1546180747..55ee65a05d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 + tensor-parallel-size: 2 + data-parallel-size: 2 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d974d3fb62..2bcc939d8d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8184,13 +8184,13 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: - isl: 8192 osl: 1024 search-space: - # 1p2d DEP4+TEP8, 5n: conc 16,64,128 MTP + # 1p2d TP2DP2+TEP8, 5n: conc 16,64,128 MTP - spec-decoding: "mtp" conc-list: [16, 64, 128] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml" @@ -8199,13 +8199,13 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: tp: 8 ep: 8 dp-attn: false - # 2p1d DEP4+DEP16, 6n: conc 512,1024 MTP + # 2p1d TP2DP2+DEP16, 6n: conc 512,1024 MTP - spec-decoding: "mtp" conc-list: [512, 1024] prefill: num-worker: 2 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml" From a36159fc0519e2b06fab2602081b8017a9b96ad4 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 19:25:56 -0400 Subject: [PATCH 4/6] fix: update GB200 fp8 MTP image to vllm/vllm-openai:v0.26.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 GB200 fp8 MTP 镜像更新为 vllm/vllm-openai:v0.26.0 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 2 +- configs/nvidia-master.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml index 7498395b18..fb3b40fd53 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + container: "vllm/vllm-openai:v0.26.0" precision: "fp8" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml index 55ee65a05d..97e85cf969 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + container: "vllm/vllm-openai:v0.26.0" precision: "fp8" dynamo: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2bcc939d8d..a839325f2e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8169,7 +8169,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: dp-attn: true minimaxm3-fp8-gb200-dynamo-vllm-mtp: - image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 + image: vllm/vllm-openai:v0.26.0 model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: gb200 From 34fec4305d2d2e86b1d26396b050cd42017cb595 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Fri, 31 Jul 2026 00:05:15 -0400 Subject: [PATCH 5/6] fix: update GB200 fp8 MTP image to nightly-6f91edf --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 2 +- configs/nvidia-master.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml index fb3b40fd53..186e9e3c63 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:v0.26.0" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" precision: "fp8" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml index 97e85cf969..7be9165961 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:v0.26.0" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" precision: "fp8" dynamo: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a839325f2e..a2e05234b7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8169,7 +8169,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: dp-attn: true minimaxm3-fp8-gb200-dynamo-vllm-mtp: - image: vllm/vllm-openai:v0.26.0 + image: vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: gb200 From 7db4551259a0ff65a6838ea3cf30531f4d1d9f46 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 1 Aug 2026 01:27:31 -0400 Subject: [PATCH 6/6] redesign GB200 MTP sweep: 1P DEP4 prefill, 1D x TP4/TP8/DEP4/DEP8/DEP16/DEP32 decode, conc 1-1024 --- ....yaml => 1p1d-dep4-dep16-eagle3-8k1k.yaml} | 14 +-- .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 106 ++++++++++++++++++ .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 106 ++++++++++++++++++ .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 106 ++++++++++++++++++ .../8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 104 +++++++++++++++++ ...1k.yaml => 1p1d-dep4-tp8-eagle3-8k1k.yaml} | 14 +-- configs/nvidia-master.yaml | 79 ++++++++++--- 7 files changed, 502 insertions(+), 27 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/{2p1d-dep4-dep16-eagle3-8k1k.yaml => 1p1d-dep4-dep16-eagle3-8k1k.yaml} (91%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml rename benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/{1p2d-dep4-tep8-eagle3-8k1k.yaml => 1p1d-dep4-tp8-eagle3-8k1k.yaml} (91%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml similarity index 91% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index 7be9165961..a6e771274e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep16-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" @@ -23,9 +23,9 @@ srun_options: resources: gpu_type: "gb200" gpus_per_node: 4 - prefill_nodes: 2 + prefill_nodes: 1 decode_nodes: 4 - prefill_workers: 2 + prefill_workers: 1 decode_workers: 1 gpus_per_prefill: 4 gpus_per_decode: 16 @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 2 - data-parallel-size: 2 + tensor-parallel-size: 1 + data-parallel-size: 4 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true @@ -72,7 +72,7 @@ backend: language-model-only: true kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 100 + stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 @@ -100,7 +100,7 @@ benchmark: type: "sa-bench" isl: 8192 osl: 1024 - concurrencies: "512x1024" + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" req_rate: "inf" random_range_ratio: 0.8 use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml new file mode 100644 index 0000000000..78eab6394e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep32-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 8 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 32 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml new file mode 100644 index 0000000000..849eed6b75 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep4-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..8f41b6b052 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml new file mode 100644 index 0000000000..a582ac07f4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -0,0 +1,104 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tp4-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml similarity index 91% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index 186e9e3c63..bc57a8afd1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tp8-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" @@ -24,9 +24,9 @@ resources: gpu_type: "gb200" gpus_per_node: 4 prefill_nodes: 1 - decode_nodes: 4 + decode_nodes: 2 prefill_workers: 1 - decode_workers: 2 + decode_workers: 1 gpus_per_prefill: 4 gpus_per_decode: 8 @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 2 - data-parallel-size: 2 + tensor-parallel-size: 1 + data-parallel-size: 4 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true @@ -92,13 +92,13 @@ backend: stream-interval: 32 max-num-seqs: 1024 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 + max-cudagraph-capture-size: 2048 benchmark: type: "sa-bench" isl: 8192 osl: 1024 - concurrencies: "16x64x128" + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" req_rate: "inf" random_range_ratio: 0.8 use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a2e05234b7..e9efa14b79 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8184,37 +8184,90 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: - isl: 8192 osl: 1024 search-space: - # 1p2d TP2DP2+TEP8, 5n: conc 16,64,128 MTP - spec-decoding: "mtp" - conc-list: [16, 64, 128] + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml" decode: - num-worker: 2 + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml" + decode: + num-worker: 1 tp: 8 ep: 8 dp-attn: false - # 2p1d TP2DP2+DEP16, 6n: conc 512,1024 MTP - spec-decoding: "mtp" - conc-list: [512, 1024] + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/Qwen3.5-397B-A17B-NVFP4