diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..186e9e3c63 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -0,0 +1,104 @@ +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 2 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16x64x128" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml new file mode 100644 index 0000000000..7be9165961 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 2 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..a2e05234b7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8060,7 +8060,7 @@ qwen3.5-fp4-b200-trt: - { tp: 8, ep: 8, dp-attn: true, conc-list: [256, 512, 1024] } # MiniMax-M3 GB200 disagg sweep — adapted from NV B300 PR #1863. -# All prefill DEP4 (TP1 DP4 EP, 4 GPU/worker). Decode: Marlin, TEP8, DEP8, TEP4. +# All prefill DEP4 (TP1 DP4 EP, 4 GPU/worker). Decode: TEP8, DEP16, TEP4. # 4 GPU/node (GB200 NVL72). FLASHINFER attention with FP8 KV cache. minimaxm3-fp8-gb200-dynamo-vllm: image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 @@ -8069,7 +8069,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: runner: gb200 precision: fp8 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260614" } + router: { name: dynamo-router, version: "1.3.0.dev20260710" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8078,7 +8078,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: - isl: 8192 osl: 1024 search-space: - # 1p2d DEP4+DEP8, 5n: conc 512 + # 1p1d DEP4+DEP16, 5n: conc 512 - conc-list: [512] prefill: num-worker: 1 @@ -8123,7 +8123,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 8 dp-attn: false - # 2p2d DEP4+DEP8, 6n: conc 512,1024 + # 2p1d DEP4+DEP16, 6n: conc 512,1024 - conc-list: [512, 1024] prefill: num-worker: 2 @@ -8138,7 +8138,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true - # 3p2d DEP4+DEP8, 7n: conc 4096 + # 3p1d DEP4+DEP16, 7n: conc 4096 - conc-list: [4096] prefill: num-worker: 3 @@ -8153,7 +8153,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true - # 5p2d DEP4+DEP8, 9n: conc 4096 + # 5p1d DEP4+DEP16, 9n: conc 4096 - conc-list: [4096] prefill: num-worker: 5 @@ -8168,6 +8168,53 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true +minimaxm3-fp8-gb200-dynamo-vllm-mtp: + image: vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: gb200 + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260710" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1p2d TP2DP2+TEP8, 5n: conc 16,64,128 MTP + - spec-decoding: "mtp" + conc-list: [16, 64, 128] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false + # 2p1d TP2DP2+DEP16, 6n: conc 512,1024 MTP + - spec-decoding: "mtp" + conc-list: [512, 1024] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..d3544cb162 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,10 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - minimaxm3-fp8-gb200-dynamo-vllm-mtp + description: + - "Add MiniMax M3 MXFP8 GB200 Dynamo-vLLM disaggregated EAGLE3 MTP recipes" + - "Fix comment errors in minimaxm3-fp8-gb200-dynamo-vllm (Xp2d DEP8 -> Xp1d DEP16, remove phantom Marlin entry, bump router to 1.3.0.dev20260710)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2432