From e46678fdd87c5a923ed78460e71988705e848e53 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 28 Jul 2026 10:47:06 -0700 Subject: [PATCH 1/4] Add DSv4-Pro FP4 B200 disaggregated Dynamo vLLM MTP config - New dsv4-fp4-b200-dynamo-vllm-mtp entry for 8k/1k MTP disagg on cluster:b200-dgxc - Five srt-slurm recipes covering 1P1D through 3P1D topologies - Pin the DSV4 Dynamo vLLM launch to the srt-slurm main branch --- .../8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml | 130 +++++++++++++++++ .../8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml | 115 +++++++++++++++ .../8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml | 131 ++++++++++++++++++ .../8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml | 130 +++++++++++++++++ .../8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml | 130 +++++++++++++++++ configs/nvidia-master.yaml | 88 ++++++++++++ perf-changelog.yaml | 8 ++ runners/launch_b200-dgxc.sh | 3 +- 8 files changed, 733 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..079fda5695 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,130 @@ +name: "svf-vllm-disagg-b200-1p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh +health_check: + max_attempts: 90 + interval_seconds: 10 +sbatch_directives: + segment: "1" +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: false + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml new file mode 100644 index 0000000000..bf9cbb9f60 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml @@ -0,0 +1,115 @@ +name: "svf-vllm-disagg-b200-1p1d-dep8-tp8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 9280 + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 16384 + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + max-num-seqs: 256 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x8x16" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..fd855131f0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml @@ -0,0 +1,131 @@ +name: "svf-vllm-disagg-b200-1p2d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 128 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128x256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..8154db9d71 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,130 @@ +name: "svf-vllm-disagg-b200-2p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..924f65643b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,130 @@ +name: "svf-vllm-disagg-b200-3p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh +health_check: + max_attempts: 90 + interval_seconds: 10 +sbatch_directives: + segment: "1" +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 3 + decode_nodes: 1 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":1,"rejection_sample_method":"synthetic","synthetic_acceptance_length":1.79}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":1,"rejection_sample_method":"synthetic","synthetic_acceptance_length":1.79}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8cc99a8a0b..eb2808b17f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5775,6 +5775,94 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 16 dp-attn: true +dsv4-fp4-b200-dynamo-vllm-mtp: + image: vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: true + kv-p2p-transfer: mooncake + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # B200 MTP disagg recipes. Each worker maps to one full 8-GPU B200 node + # (prefill dep8; decode tp8, except the low-latency tp8 variant). + - spec-decoding: "mtp" + conc-list: [1, 8, 16] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [128, 256] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml" + decode: + num-worker: 2 + tp: 1 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1024] + prefill: + num-worker: 2 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [4096] + prefill: + num-worker: 3 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + dsv4-fp4-gb300-dynamo-trt: image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0381848924..60beba93d4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5173,3 +5173,11 @@ - "Bring-up validated in run 30326393603: all 12 configs green, zero ServerDisconnectedError after the keep-alive fix. GPU KV resolves to 42.23 GiB / 3,249,215 tokens, i.e. ~3.1 max-length requests, against MAX_NUM_SEQS = 2*CONC." - "Measured behaviour: below conc 8 the GPU-resident and DRAM arms are within run-to-run noise (1-5%). At conc 16 and 24 the GPU-resident arm thrashes -- prefix cache hit rate 2.7%, TTFT p50 86s and 191s, 49.6 and 54.9 output tok/s -- because the working set exceeds GPU KV and prefixes are recomputed. The DRAM arm holds TTFT p50 0.85s and 6.2s for 245.0 and 260.6 output tok/s (4-5x), with the CPU tier serving a 62% external prefix cache hit rate at conc 24. The high-conc GPU-resident points are retained deliberately as the honest baseline that makes the offload gain legible." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2386 + +- config-keys: + - dsv4-fp4-b200-dynamo-vllm-mtp + description: + - "Add DeepSeek-V4-Pro FP4 B200 disaggregated Dynamo vLLM MTP benchmarks for 8k/1k" + - "Add five srt-slurm recipes covering low-latency and high-throughput topologies from 1P1D through 3P1D" + - "Pin the DSV4 Dynamo vLLM launch to the srt-slurm main branch" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a276644575..5d26b66227 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -108,9 +108,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 - git checkout aflowers/vllm-gb200-v0.20.0 mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then From d540d373548b6fb753e08e8bf86a13f10c2c69c2 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 28 Jul 2026 10:47:34 -0700 Subject: [PATCH 2/4] Update perf-changelog pr-link for #2390 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 60beba93d4..cf557fa291 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5180,4 +5180,4 @@ - "Add DeepSeek-V4-Pro FP4 B200 disaggregated Dynamo vLLM MTP benchmarks for 8k/1k" - "Add five srt-slurm recipes covering low-latency and high-throughput topologies from 1P1D through 3P1D" - "Pin the DSV4 Dynamo vLLM launch to the srt-slurm main branch" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2390 From de214edaac3aac18c5a62dc10e97be9a00d6a660 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 30 Jul 2026 17:12:33 -0700 Subject: [PATCH 3/4] update eval injector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：更新评估注入器。 --- runners/inject_synthetic_acceptance.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/runners/inject_synthetic_acceptance.py b/runners/inject_synthetic_acceptance.py index 4bda75a9a4..80d2740e79 100644 --- a/runners/inject_synthetic_acceptance.py +++ b/runners/inject_synthetic_acceptance.py @@ -47,6 +47,11 @@ def _log(msg): print(f"[Synthetic AR] {msg}") +def _enabled(name: str) -> bool: + """Return whether an environment flag is explicitly enabled.""" + return os.environ.get(name, "false").strip().lower() == "true" + + def _yaml_key(model_prefix): return MODEL_PREFIX_TO_YAML_KEY.get(model_prefix, model_prefix) @@ -120,7 +125,11 @@ def _resolve_al(config_text, injector, ref_yaml): def inject(config_file, framework): - if os.environ.get("SYNTHETIC_ACCEPTANCE", "false") != "true": + if _enabled("EVAL_ONLY"): + print("[Synthetic AL] EVAL_ONLY=true: keeping real MTP recipe") + return 0 + + if not _enabled("SYNTHETIC_ACCEPTANCE"): return 0 injector = get_injector(framework) @@ -144,8 +153,10 @@ def inject(config_file, framework): new_content, count = injector.rewrite(content, al, _log) if count == 0: - _log("WARNING: No speculative-config entries found to modify; leaving recipe unchanged") - return 0 + sys.exit( + "ERROR: SYNTHETIC_ACCEPTANCE=true but no speculative-config " + f"entries were found in {config_file}" + ) with open(config_file, "w") as f: f.write(new_content) From c086cc2efb0879d8e83a644640782820ef904b12 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Fri, 31 Jul 2026 11:59:36 -0700 Subject: [PATCH 4/4] Switch dsv4-fp4-b200-dynamo-vllm-mtp runner to b200-new --- configs/nvidia-master.yaml | 2 +- runners/launch_b200-nscale-slurm.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 runners/launch_b200-nscale-slurm.sh diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 476cd87faf..78260b442e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6151,7 +6151,7 @@ dsv4-fp4-b200-dynamo-vllm-mtp: image: vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: b200-multinode + runner: b200-new precision: fp4 framework: dynamo-vllm multinode: true diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh new file mode 120000 index 0000000000..f45b7edcb3 --- /dev/null +++ b/runners/launch_b200-nscale-slurm.sh @@ -0,0 +1 @@ +launch_b200-dgxc.sh \ No newline at end of file