diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep4-tp4-c1-c16-8k1k.yaml similarity index 93% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep4-tp4-c1-c16-8k1k.yaml index d28f594bae..cfb93de81a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep4-tp4-c1-c16-8k1k.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-b200-1p1d-fp4-dep2-tp4-8k1k" +name: "minimax-m3-vllm-disagg-b200-1p1d-fp4-dep4-tp4-8k1k" model: path: "nvidia/MiniMax-M3-NVFP4" container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" @@ -10,9 +10,9 @@ resources: decode_nodes: 1 prefill_workers: 1 decode_workers: 1 - gpus_per_prefill: 2 + gpus_per_prefill: 4 gpus_per_decode: 4 -dynamo: {install: true, version: 1.3.0.dev20260710} +dynamo: {install: true, version: 1.3.0.dev20260713} frontend: {type: dynamo, enable_multiple_frontends: false} backend: type: vllm @@ -25,7 +25,7 @@ backend: served-model-name: nvidia/MiniMax-M3-NVFP4 no-enable-flashinfer-autotune: true tensor-parallel-size: 1 - data-parallel-size: 2 + data-parallel-size: 4 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep4-tp4-c64-8k1k.yaml similarity index 95% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep4-tp4-c64-8k1k.yaml index 7545f73b29..04ad968cc7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep4-tp4-c64-8k1k.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-b200-1p2d-fp4-dep2-tp4-c64-8k1k" +name: "minimax-m3-vllm-disagg-b200-1p2d-fp4-dep4-tp4-c64-8k1k" model: path: "nvidia/MiniMax-M3-NVFP4" container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" @@ -7,13 +7,13 @@ resources: gpu_type: "b200" prefill_nodes: 1 prefill_workers: 1 - gpus_per_prefill: 2 + gpus_per_prefill: 4 decode_nodes: 2 decode_workers: 2 spread_workers: true gpus_per_decode: 4 gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} +dynamo: {install: true, version: 1.3.0.dev20260713} frontend: type: dynamo enable_multiple_frontends: false @@ -70,7 +70,7 @@ backend: tensor-parallel-size: 1 pipeline-parallel-size: 1 enable-expert-parallel: true - data-parallel-size: 2 + data-parallel-size: 4 data-parallel-rpc-port: 13345 data-parallel-hybrid-lb: true max-model-len: 9472 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep4-tp4-c128-8k1k.yaml similarity index 95% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep4-tp4-c128-8k1k.yaml index 4614abeae0..eb990a3240 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep4-tp4-c128-8k1k.yaml @@ -1,19 +1,19 @@ -name: "minimax-m3-vllm-disagg-b200-2p2d-fp4-dep2-tp4-c128-8k1k" +name: "minimax-m3-vllm-disagg-b200-2p2d-fp4-dep4-tp4-c128-8k1k" model: path: "nvidia/MiniMax-M3-NVFP4" container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" precision: "fp4" resources: gpu_type: "b200" - prefill_nodes: 2 + prefill_nodes: 1 prefill_workers: 2 - gpus_per_prefill: 2 + gpus_per_prefill: 4 decode_nodes: 2 decode_workers: 2 spread_workers: true gpus_per_decode: 4 gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} +dynamo: {install: true, version: 1.3.0.dev20260713} frontend: type: dynamo enable_multiple_frontends: false @@ -70,7 +70,7 @@ backend: tensor-parallel-size: 1 pipeline-parallel-size: 1 enable-expert-parallel: true - data-parallel-size: 2 + data-parallel-size: 4 data-parallel-rpc-port: 13345 data-parallel-hybrid-lb: true max-model-len: 9472 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c256-8k1k.yaml similarity index 95% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c256-8k1k.yaml index 8cfb3eb300..d7662e1b2e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c256-8k1k.yaml @@ -1,19 +1,19 @@ -name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c256-8k1k" +name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep4-tp4-c256-8k1k" model: path: "nvidia/MiniMax-M3-NVFP4" container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" precision: "fp4" resources: gpu_type: "b200" - prefill_nodes: 3 + prefill_nodes: 2 prefill_workers: 3 - gpus_per_prefill: 2 + gpus_per_prefill: 4 decode_nodes: 2 decode_workers: 2 spread_workers: true gpus_per_decode: 4 gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} +dynamo: {install: true, version: 1.3.0.dev20260713} frontend: type: dynamo enable_multiple_frontends: false @@ -70,7 +70,7 @@ backend: tensor-parallel-size: 1 pipeline-parallel-size: 1 enable-expert-parallel: true - data-parallel-size: 2 + data-parallel-size: 4 data-parallel-rpc-port: 13345 data-parallel-hybrid-lb: true max-model-len: 9472 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c512-8k1k.yaml similarity index 95% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c512-8k1k.yaml index bf9f0d7e59..603bfb40ed 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c512-8k1k.yaml @@ -1,19 +1,19 @@ -name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c512-8k1k" +name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep4-tp4-c512-8k1k" model: path: "nvidia/MiniMax-M3-NVFP4" container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" precision: "fp4" resources: gpu_type: "b200" - prefill_nodes: 3 + prefill_nodes: 2 prefill_workers: 3 - gpus_per_prefill: 2 + gpus_per_prefill: 4 decode_nodes: 2 decode_workers: 2 spread_workers: true gpus_per_decode: 4 gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} +dynamo: {install: true, version: 1.3.0.dev20260713} frontend: type: dynamo enable_multiple_frontends: false @@ -70,7 +70,7 @@ backend: tensor-parallel-size: 1 pipeline-parallel-size: 1 enable-expert-parallel: true - data-parallel-size: 2 + data-parallel-size: 4 data-parallel-rpc-port: 13345 data-parallel-hybrid-lb: true max-model-len: 9472 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep4-tp4-c1024-8k1k.yaml similarity index 95% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep4-tp4-c1024-8k1k.yaml index ff366b74eb..8b6965818e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep4-tp4-c1024-8k1k.yaml @@ -1,19 +1,19 @@ -name: "minimax-m3-vllm-disagg-b200-4p2d-fp4-dep2-tp4-c1024-8k1k" +name: "minimax-m3-vllm-disagg-b200-4p2d-fp4-dep4-tp4-c1024-8k1k" model: path: "nvidia/MiniMax-M3-NVFP4" container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" precision: "fp4" resources: gpu_type: "b200" - prefill_nodes: 4 + prefill_nodes: 2 prefill_workers: 4 - gpus_per_prefill: 2 + gpus_per_prefill: 4 decode_nodes: 2 decode_workers: 2 spread_workers: true gpus_per_decode: 4 gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} +dynamo: {install: true, version: 1.3.0.dev20260713} frontend: type: dynamo enable_multiple_frontends: false @@ -70,7 +70,7 @@ backend: tensor-parallel-size: 1 pipeline-parallel-size: 1 enable-expert-parallel: true - data-parallel-size: 2 + data-parallel-size: 4 data-parallel-rpc-port: 13345 data-parallel-hybrid-lb: true max-model-len: 9472 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml new file mode 100644 index 0000000000..44ae8cdc3d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -0,0 +1,93 @@ +name: "minimax-m3-vllm-disagg-b200-1p1d-dep4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 0 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +dynamo: + install: true + version: 1.3.0.dev20260713 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + decode_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + enforce-eager: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-batched-tokens: 16384 + max-num-seqs: 256 + + decode: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 256 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..a99f24daba --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -0,0 +1,93 @@ +name: "minimax-m3-vllm-disagg-b200-1p1d-dep4-tp8-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +dynamo: + install: true + version: 1.3.0.dev20260713 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + decode_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + enforce-eager: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-batched-tokens: 16384 + max-num-seqs: 256 + + decode: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 128 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 128 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp4-eagle3-8k1k.yaml new file mode 100644 index 0000000000..4118e0b805 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp4-eagle3-8k1k.yaml @@ -0,0 +1,93 @@ +name: "minimax-m3-vllm-disagg-b200-1p2d-dep4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +dynamo: + install: true + version: 1.3.0.dev20260713 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + decode_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + enforce-eager: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-batched-tokens: 16384 + max-num-seqs: 256 + + decode: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 128 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 128 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..61be00084c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp8-eagle3-8k1k.yaml @@ -0,0 +1,93 @@ +name: "minimax-m3-vllm-disagg-b200-1p2d-dep4-tp8-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +dynamo: + install: true + version: 1.3.0.dev20260713 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + decode_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + enforce-eager: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-batched-tokens: 16384 + max-num-seqs: 256 + + decode: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 64 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 64 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp4-eagle3-8k1k.yaml new file mode 100644 index 0000000000..4b8cf224f5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp4-eagle3-8k1k.yaml @@ -0,0 +1,93 @@ +name: "minimax-m3-vllm-disagg-b200-1p4d-dep4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 4 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +dynamo: + install: true + version: 1.3.0.dev20260713 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + decode_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + enforce-eager: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-batched-tokens: 16384 + max-num-seqs: 256 + + decode: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 64 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 64 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..3351cfb9bf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp8-eagle3-8k1k.yaml @@ -0,0 +1,93 @@ +name: "minimax-m3-vllm-disagg-b200-1p4d-dep4-tp8-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 4 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +dynamo: + install: true + version: 1.3.0.dev20260713 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + decode_environment: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + enforce-eager: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-batched-tokens: 16384 + max-num-seqs: 256 + + decode: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 32 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 32 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..d13db9eaee 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7393,7 +7393,7 @@ minimaxm3-fp4-b200-dynamo-vllm: runner: b200-multinode precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7405,11 +7405,11 @@ minimaxm3-fp4-b200-dynamo-vllm: - conc-list: [1, 4, 8, 16] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep4-tp4-c1-c16-8k1k.yaml" decode: num-worker: 1 tp: 4 @@ -7418,11 +7418,11 @@ minimaxm3-fp4-b200-dynamo-vllm: - conc-list: [64] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep4-tp4-c64-8k1k.yaml" decode: num-worker: 2 tp: 4 @@ -7431,11 +7431,11 @@ minimaxm3-fp4-b200-dynamo-vllm: - conc-list: [128] prefill: num-worker: 2 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep4-tp4-c128-8k1k.yaml" decode: num-worker: 2 tp: 4 @@ -7444,11 +7444,11 @@ minimaxm3-fp4-b200-dynamo-vllm: - conc-list: [256] prefill: num-worker: 3 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c256-8k1k.yaml" decode: num-worker: 2 tp: 4 @@ -7457,11 +7457,11 @@ minimaxm3-fp4-b200-dynamo-vllm: - conc-list: [512] prefill: num-worker: 3 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep4-tp4-c512-8k1k.yaml" decode: num-worker: 2 tp: 4 @@ -7470,20 +7470,117 @@ minimaxm3-fp4-b200-dynamo-vllm: - conc-list: [1024] prefill: num-worker: 4 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep4-tp4-c1024-8k1k.yaml" decode: num-worker: 2 tp: 4 ep: 1 dp-attn: false -# MiniMax-M3 NVFP4 disagg sweep on the same B300 topology matrix as the MXFP8 -# baseline above. The image includes vLLM PR #46380, so no runtime patch is -# needed. +minimaxm3-fp4-b200-dynamo-vllm-mtp: + image: vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: b200-multinode + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260713" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp4-eagle3-8k1k.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-dep4-tp8-eagle3-8k1k.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp4-eagle3-8k1k.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-dep4-tp8-eagle3-8k1k.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false minimaxm3-fp8-b300-dynamo-vllm: image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -7774,7 +7871,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp: runner: b300 precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7862,7 +7959,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp-legacy-dep4: runner: b300 precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..f966216da3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,9 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - minimaxm3-fp4-b200-dynamo-vllm-mtp + description: + - "Add MiniMax M3 NVFP4 B200 Dynamo-vLLM disaggregated EAGLE3 recipes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2431