diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..9a382b0dae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,131 @@ +name: "svf-vllm-disagg-b200-1p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +health_check: + max_attempts: 90 + interval_seconds: 10 +sbatch_directives: + segment: "1" +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: false + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml new file mode 100644 index 0000000000..f5096f712a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml @@ -0,0 +1,116 @@ +name: "svf-vllm-disagg-b200-1p1d-dep8-tp8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 9280 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 16384 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + max-num-seqs: 256 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x8x16" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..219fb9c491 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml @@ -0,0 +1,132 @@ +name: "svf-vllm-disagg-b200-1p2d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 128 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128x256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..0e96fff2d5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,131 @@ +name: "svf-vllm-disagg-b200-2p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..b07bbed4c9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,131 @@ +name: "svf-vllm-disagg-b200-3p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +health_check: + max_attempts: 90 + interval_seconds: 10 +sbatch_directives: + segment: "1" +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 3 + decode_nodes: 1 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":1}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":1}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..78260b442e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6147,6 +6147,107 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 16 dp-attn: true +dsv4-fp4-b200-dynamo-vllm-mtp: + image: vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b200-new + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: true + kv-p2p-transfer: nixl + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # B200 MTP disagg recipes. Each worker maps to one full 8-GPU B200 node + # (prefill dep8; decode tp8, except the low-latency tp8 variant). + # SYNTHETIC_ACCEPTANCE is a throughput-only control (injected into the + # recipe by runners/inject_synthetic_acceptance.py); EVAL_ONLY runs keep + # the checked-in real-MTP recipe so verification is measured normally. + - spec-decoding: "mtp" + conc-list: [1, 8, 16] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [128, 256] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml" + decode: + num-worker: 2 + tp: 1 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1024] + prefill: + num-worker: 2 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [4096] + prefill: + num-worker: 3 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=1.79" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + dsv4-fp4-gb300-dynamo-trt: image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..95a4f5a8aa 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,9 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - dsv4-fp4-b200-dynamo-vllm-mtp + description: + - "Add DeepSeek-V4-Pro FP4 B200 disaggregated Dynamo vLLM MTP benchmarks for 8k/1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2390 diff --git a/runners/inject_synthetic_acceptance.py b/runners/inject_synthetic_acceptance.py index 4bda75a9a4..80d2740e79 100644 --- a/runners/inject_synthetic_acceptance.py +++ b/runners/inject_synthetic_acceptance.py @@ -47,6 +47,11 @@ def _log(msg): print(f"[Synthetic AR] {msg}") +def _enabled(name: str) -> bool: + """Return whether an environment flag is explicitly enabled.""" + return os.environ.get(name, "false").strip().lower() == "true" + + def _yaml_key(model_prefix): return MODEL_PREFIX_TO_YAML_KEY.get(model_prefix, model_prefix) @@ -120,7 +125,11 @@ def _resolve_al(config_text, injector, ref_yaml): def inject(config_file, framework): - if os.environ.get("SYNTHETIC_ACCEPTANCE", "false") != "true": + if _enabled("EVAL_ONLY"): + print("[Synthetic AL] EVAL_ONLY=true: keeping real MTP recipe") + return 0 + + if not _enabled("SYNTHETIC_ACCEPTANCE"): return 0 injector = get_injector(framework) @@ -144,8 +153,10 @@ def inject(config_file, framework): new_content, count = injector.rewrite(content, al, _log) if count == 0: - _log("WARNING: No speculative-config entries found to modify; leaving recipe unchanged") - return 0 + sys.exit( + "ERROR: SYNTHETIC_ACCEPTANCE=true but no speculative-config " + f"entries were found in {config_file}" + ) with open(config_file, "w") as f: f.write(new_content) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index be68ce073b..ed24a61d13 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -300,12 +300,21 @@ EOF exit 1 fi + # Strip any :override[N] selector so sed and the injector operate on the file. + CONFIG_PATH="${CONFIG_FILE%%:*}" + # Override the job name in the config file with the runner name - sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "${CONFIG_FILE%%:*}" + sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" # Bump recipe health-check timeout from 360×10s=3600s to 720×10s=7200s # so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time. - # Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix. - sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" + sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "$CONFIG_PATH" + + # Optionally inject synthetic acceptance into the recipe's speculative-config + # when SYNTHETIC_ACCEPTANCE=true (no-op otherwise). Must run after the name + # override and before srtctl apply so the rendered job picks it up. Fail + # fast so a broken opt-in never reaches srtctl with an unrewritten recipe. + python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" || exit 1 SRTCTL_PREFLIGHT_ARGS=() # Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh new file mode 120000 index 0000000000..f45b7edcb3 --- /dev/null +++ b/runners/launch_b200-nscale-slurm.sh @@ -0,0 +1 @@ +launch_b200-dgxc.sh \ No newline at end of file