Standardize srt-slurm on v1.0.36

benchmarks/benchmark_lib.sh

@@ -327,7 +327,7 @@ wait_for_server_ready() {
 }
 
 # Run benchmark serving with standardized parameters
-# All parameters are required except --endpoint, --use-chat-template, --dsv4, and --trust-remote-code
+# All parameters are required except the optional parameters listed below.
 # Parameters:
 #   --model: Model name
 #   --port: Server port
@@ -346,9 +346,14 @@ wait_for_server_ready() {
 #           template. Implies --use-chat-template.
 #   --trust-remote-code: Optional flag to trust remote code from HuggingFace
 #   --server-pid: Optional server process ID to monitor during benchmark
+#   --tokenizer: Optional tokenizer name/path override
+#   --tokenizer-mode: Optional tokenizer implementation mode
+#   --host: Optional server host (default: 0.0.0.0)
+#   --request-rate: Optional request rate (default: inf)
+#   --num-warmups: Optional warmup request count (default: 2 * concurrency)
 run_benchmark_serving() {
     # In eval-only mode, skip the throughput benchmark entirely.
-    if [ "${EVAL_ONLY}" = "true" ]; then
+    if [ "${EVAL_ONLY:-false}" = "true" ]; then
         echo "EVAL_ONLY mode: skipping throughput benchmark"
         return 0
     fi
@@ -372,6 +377,9 @@ run_benchmark_serving() {
     local server_pid=""
     local tokenizer=""
     local tokenizer_mode=""
+    local host="0.0.0.0"
+    local request_rate="inf"
+    local num_warmups=""
 
     while [[ $# -gt 0 ]]; do
         case $1 in
@@ -448,6 +456,18 @@ run_benchmark_serving() {
                 tokenizer_mode="$2"
                 shift 2
                 ;;
+            --host)
+                host="$2"
+                shift 2
+                ;;
+            --request-rate)
+                request_rate="$2"
+                shift 2
+                ;;
+            --num-warmups)
+                num_warmups="$2"
+                shift 2
+                ;;
             *)
                 echo "Unknown parameter: $1"
                 return 1
@@ -500,6 +520,9 @@ run_benchmark_serving() {
     if [[ -z "$workspace_dir" ]]; then
         workspace_dir=$(pwd)
     fi
+    if [[ -z "$num_warmups" ]]; then
+        num_warmups="$((2 * max_concurrency))"
+    fi
 
     # Profiling support: when PROFILE=1, ensure profiler dir exists, add --profile flag,
     # and cap num_prompts to keep traces small.
@@ -518,18 +541,18 @@ run_benchmark_serving() {
         python3 "$workspace_dir/utils/bench_serving/benchmark_serving.py"
         --model "$model"
         --backend "$backend"
-        --base-url "http://0.0.0.0:$port"
+        --base-url "http://${host}:$port"
         --dataset-name random
         --random-input-len "$input_len"
         --random-output-len "$output_len"
         --random-range-ratio "$random_range_ratio"
         --num-prompts "$num_prompts"
         --max-concurrency "$max_concurrency"
-        --request-rate inf
+        --request-rate "$request_rate"
         --ignore-eos
         "${profile_flag[@]}"
         --save-result
-        --num-warmups "$((2 * max_concurrency))" \
+        --num-warmups "$num_warmups" \
         --percentile-metrics 'ttft,tpot,itl,e2el'
         --result-dir "$result_dir"
         --result-filename "$result_filename.json"

benchmarks/multi_node/srt-slurm-recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml

@@ -0,0 +1,154 @@
+name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx10dep4_gen1dep16_batch64_eplb384_mtp3"
+
+# ctx: 10 prefill workers, TP4/EP4, EPLB: num_slots=384
+# gen: 1 decode worker, TP16/EP16, enable_attention_dp=true, max_batch=64
+# concurrency: 1229
+
+model:
+  path: "deepseek-ai/DeepSeek-V4-Pro"
+  container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1"
+  precision: "fp4"
+
+extra_mount:
+  - "tmpfs:/dev/shm:size=100%"
+  - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs"
+resources:
+  gpu_type: "gb300"
+
+  prefill_nodes: 10
+  prefill_workers: 10
+  gpus_per_prefill: 4
+
+  decode_workers: 1
+  decode_nodes: 4
+  gpus_per_decode: 16
+
+  gpus_per_node: 4
+
+backend:
+  type: trtllm
+
+  prefill_environment:
+    TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1"
+    UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp"
+    PYTHONUNBUFFERED: "1"
+    ENROOT_ALLOW_DEV: "yes"
+    MIMALLOC_PURGE_DELAY: "-1"
+    NCCL_GRAPH_MIXING_SUPPORT: "0"
+    PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core"
+    PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
+    TLLM_LOG_LEVEL: "INFO"
+    TRTLLM_ENABLE_PDL: "1"
+    TRTLLM_SERVER_DISABLE_GC: "1"
+    TRTLLM_WORKER_DISABLE_GC: "1"
+
+  decode_environment:
+    UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp"
+    PYTHONUNBUFFERED: "1"
+    ENROOT_ALLOW_DEV: "yes"
+    MIMALLOC_PURGE_DELAY: "0"
+    NCCL_GRAPH_MIXING_SUPPORT: "0"
+    PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core"
+    TLLM_LOG_LEVEL: "INFO"
+    TRTLLM_ENABLE_PDL: "1"
+    TRTLLM_SERVER_DISABLE_GC: "1"
+    TRTLLM_WORKER_DISABLE_GC: "1"
+
+  trtllm_config:
+    prefill:
+      cache_transceiver_config:
+        backend: NIXL
+        kv_transfer_timeout_ms: 600000
+        max_tokens_in_buffer: 8192
+        transceiver_runtime: PYTHON
+      cuda_graph_config: null
+      disable_overlap_scheduler: true
+      enable_attention_dp: true
+      kv_cache_config:
+        dtype: fp8
+        enable_block_reuse: false
+        free_gpu_memory_fraction: 0.5
+        tokens_per_block: 128
+      max_batch_size: 2
+      max_num_tokens: 8192
+      max_seq_len: 8232
+      moe_config:
+        backend: TRTLLM
+        load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml"
+      moe_expert_parallel_size: 4
+      pipeline_parallel_size: 1
+      print_iter_log: true
+      trust_remote_code: true
+      speculative_config:
+        decoding_type: MTP
+        max_draft_len: 3
+      tensor_parallel_size: 4
+
+    decode:
+      cache_transceiver_config:
+        backend: NIXL
+        kv_transfer_timeout_ms: 600000
+        max_tokens_in_buffer: 8192
+        transceiver_runtime: PYTHON
+      cuda_graph_config:
+        batch_sizes:
+          - 1
+          - 2
+          - 4
+          - 8
+          - 16
+          - 24
+          - 32
+          - 40
+          - 48
+          - 56
+          - 64
+        enable_padding: true
+      enable_attention_dp: true
+      enable_lm_head_tp_in_adp: true
+      kv_cache_config:
+        dtype: fp8
+        enable_block_reuse: false
+        free_gpu_memory_fraction: 0.7
+        tokens_per_block: 128
+      max_batch_size: 64
+      max_num_tokens: 256
+      max_seq_len: 9256
+      moe_config:
+        backend: MEGAMOE_DEEPGEMM
+        load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep16_slots384.yaml"
+        use_low_precision_moe_combine: true
+      moe_expert_parallel_size: 16
+      num_postprocess_workers: 4
+      pipeline_parallel_size: 1
+      print_iter_log: true
+      trust_remote_code: true
+      speculative_config:
+        decoding_type: MTP
+        max_draft_len: 3
+      stream_interval: 100
+      tensor_parallel_size: 16
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/srt_bench_serving_entrypoint.sh
+  isl: 8192
+  osl: 1024
+  concurrencies: "1229"
+  req_rate: "inf"
+  num_prompts_mult: 10
+  num_warmup_mult: 20
+  custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer"
+  use_chat_template: true
+
+frontend:
+  type: "dynamo"
+  enable_multiple_frontends: true
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+dynamo:
+  request_plane: "tcp"
+  install: false