diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index dfb3f7aafc..8a1436fe15 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1852,6 +1852,9 @@ build_replay_cmd() { # this as a ``min(cap, available)`` ceiling, not a target — see # semianalysis_cc_traces_weka.py). REPLAY_CMD+=" --num-dataset-entries 393" + if declare -p AIPERF_TRACE_IDLE_GAP_CAP_SECONDS >/dev/null 2>&1; then + REPLAY_CMD+=" --trace-idle-gap-cap-seconds $AIPERF_TRACE_IDLE_GAP_CAP_SECONDS" + fi # 1-second timeslices on the server-metrics scrape so the post-run # plotter has per-window time series (KV usage, cache hit rate, # throughput, etc.). Matches kv-cache-tester's poll_interval=1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml new file mode 100644 index 0000000000..d871a1c498 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml @@ -0,0 +1,94 @@ +name: "kimik3-vllm-agg-h200-tp16dp2ep32-latency-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + vllm: "0.1.dev19262+gb6bbf29dd.d20260727" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + CUDA_LAUNCH_BLOCKING: "1" + GLOO_SOCKET_IFNAME: "eth0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + NCCL_SOCKET_IFNAME: "eth0" + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + PYTHONNOUSERSITE: "1" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" + VLLM_USE_V2_MODEL_RUNNER: "1" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + data-parallel-size: 2 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: marlin + attention-backend: FLASHMLA + gpu-memory-utilization: 0.975 + max-num-seqs: 5 + max-num-batched-tokens: 4096 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + enforce-eager: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + no-enable-flashinfer-autotune: true + disable-custom-all-reduce: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml new file mode 100644 index 0000000000..ba2c7f226d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml @@ -0,0 +1,95 @@ +name: "kimik3-vllm-agg-h200-tp8dp4ep32-balanced-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + vllm: "0.1.dev19262+gb6bbf29dd.d20260727" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + CUDA_LAUNCH_BLOCKING: "1" + GLOO_SOCKET_IFNAME: "eth0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + NCCL_SOCKET_IFNAME: "eth0" + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + PYTHONNOUSERSITE: "1" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" + VLLM_USE_V2_MODEL_RUNNER: "1" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: marlin + attention-backend: FLASHMLA + gpu-memory-utilization: 0.95 + max-num-seqs: 8 + max-num-batched-tokens: 4096 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + enforce-eager: true + no-async-scheduling: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + no-enable-flashinfer-autotune: true + disable-custom-all-reduce: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..8f76304fbb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml @@ -0,0 +1,96 @@ +name: "kimik3-vllm-agg-h200-tp8dp4ep32-vllm-simple-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + vllm: "0.1.dev19262+gb6bbf29dd.d20260727" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + CUDA_LAUNCH_BLOCKING: "1" + GLOO_SOCKET_IFNAME: "eth0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + NCCL_SOCKET_IFNAME: "eth0" + NCCL_CUMEM_ENABLE: "1" + PYTHONHASHSEED: "42" + PYTHONNOUSERSITE: "1" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" + VLLM_USE_V2_MODEL_RUNNER: "1" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: marlin + attention-backend: FLASHMLA + gpu-memory-utilization: 0.95 + max-num-seqs: 16 + max-num-batched-tokens: 4096 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + enforce-eager: true + no-async-scheduling: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":154250000000,"lazy_offload":false}}' + no-enable-flashinfer-autotune: true + disable-custom-all-reduce: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..efa444f0bb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1503,6 +1503,80 @@ kimik2.5-int4-h200-vllm-agentic: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] } - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } +# Kimi K3 is a 2.8T MXFP4 MoE served on H200 nodes. +# These are the three aggregated strategy classes from the official H200 +# recipe (https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=h200): +# Wider TP is latency-oriented, while progressively wider DP is balanced or +# throughput-oriented. EP32 keeps the model shards within HBM and preserves +# the native 1M context on four H200 nodes. All arms use probabilistic DSpark K2 +# with synthetic acceptance pinned to the committed golden AL of 2.51. +# The existing `mtp` matrix label covers speculative-decoding methods; the +# checked-in recipes select DSpark explicitly. The H200 runner supplies +# cluster paths while srt-slurm owns allocation, direct vLLM startup, +# readiness, and log capture. P/D disaggregation is intentionally deferred. +kimik3-fp4-h200-vllm-agentic: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:h200-dgxc + precision: fp4 + framework: vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # Latency-oriented: TP16 x DP2 with EP32 across four H200 nodes. + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] + prefill: + num-worker: 2 + tp: 16 + ep: 32 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 32 + dp-attn: true + # Balanced: TP8 x DP4 with EP32 across four H200 nodes. + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16] + prefill: + num-worker: 4 + tp: 8 + ep: 32 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 32 + dp-attn: true + # Throughput-oriented hybrid TP/DEP/EP with vLLM's host-DRAM KV tier. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] + prefill: + num-worker: 4 + tp: 8 + ep: 32 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 32 + dp-attn: true + # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html # does not have a B300-specific recipe, so this config reuses the existing # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..1cff1d1b50 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,9 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - kimik3-fp4-h200-vllm-agentic + description: + - "Add Day-0 Kimi K3 FP4 AgentX H200 performance recipes from the official vLLM guidance, using fit-safe TP/DP/EP topologies, optional SimpleCPU KV offload, and a five-minute reconstructed trace idle-gap cap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2353 diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 00c6cc4977..29c77b13cd 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -7,6 +7,8 @@ SLURM_ACCOUNT="sa-shared" set -x +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + if [[ "$IS_MULTINODE" == "true" ]]; then # MODEL_PATH: Override with pre-downloaded paths on H200 runner @@ -29,8 +31,16 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" exit 1 fi + elif [[ $FRAMEWORK == "vllm" ]]; then + if [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/models/gharunners/hf-hub-cache/Kimi-K3" + export SRT_SLURM_MODEL_PREFIX="kimik3" + else + echo "Unsupported model prefix/precision for vllm: $MODEL_PREFIX/$PRECISION" + exit 1 + fi else - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, vllm" exit 1 fi @@ -41,8 +51,14 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then + if [[ $IS_AGENTIC == "1" && $FRAMEWORK == "vllm" && $MODEL_PREFIX == "kimik3" ]]; then + git clone https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic + elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" else @@ -77,6 +93,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" + elif [[ $FRAMEWORK == "vllm" ]]; then + CONTAINER_KEY="$IMAGE" + SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" fi export ISL="$ISL" @@ -85,6 +104,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Create srtslurm.yaml for srtctl (used by both frameworks) SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" + DEFAULT_MOUNTS_BLOCK="" + if [[ "$IS_AGENTIC" == "1" ]]; then + AIPERF_MMAP_CACHE_HOST_PATH="/home/sa-shared/gharunners/ai-perf-cache" + HF_HUB_CACHE_HOST_PATH="/models/gharunners/hf-hub-cache" + mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" + DEFAULT_MOUNTS_BLOCK="default_mounts: + ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache + ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + fi echo "Creating srtslurm.yaml configuration..." cat > srtslurm.yaml </dev/null || true; exit "$rc"' EXIT INT TERM HUP - # Wait for log file to appear (also check job is still alive) - while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 - done - - # Poll for job completion in background - ( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 - done - ) & - POLL_PID=$! - - echo "Tailing LOG_FILE: $LOG_FILE" - - # Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) - tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - - wait $POLL_PID + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 set -x @@ -194,7 +201,7 @@ EOF echo "Found logs directory: $LOGS_DIR" cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" - tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . + bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" if [[ "${EVAL_ONLY:-false}" != "true" ]]; then # Find all result subdirectories diff --git a/utils/aiperf b/utils/aiperf index be758d6218..ed057829b7 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit be758d6218268171e2957fbec9d4f557275bca2d +Subproject commit ed057829b78d25d79ce6f3b87763d48fe50363f5