From 59956b05ce250391281bcc1022e32c758c69ed0c Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Mon, 27 Jul 2026 10:41:49 -0700 Subject: [PATCH 1/3] dsv4 H200 agentic: add aggregated TP8 DeepSeek-V4-Pro Pareto sweep Add one 8xH200 aggregated TP8 (EP1/DP1) DeepSeek-V4-Pro FP8 Dynamo-SGLang AgentX recipe with EAGLE MTP and HiCache, sweeping concurrency [1,2,4,8,16] over a fixed serving topology. Uses the Marlin MoE backend and Dynamo header-based session affinity (X-Dynamo-Session-ID). - New recipe agg-h200-tp8-mtp-kvoffload.yaml + master key dsv4-fp8-h200-dynamo-sglang-agentic-agg (decode num-worker 0 so per-GPU accounting reflects the single aggregated TP8 worker). - benchmark_lib.sh: skip the legacy nvext conv-aware CLI routing when a recipe opts into the header path (AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true); the existing AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING opt-out and default are unchanged. - launch_h200-dgxc-slurm.sh: dsv4 fp8 model-path routing, srt-slurm v1.0.10 overlay for the agentic recipe, on-demand SGLang/nginx squash imports, and AgentX dataset / HF caches mounted into the multi-node agentic path. --- benchmarks/benchmark_lib.sh | 6 +- .../agentic/agg-h200-tp8-mtp-kvoffload.yaml | 115 ++++++++++++++++++ configs/nvidia-master.yaml | 35 ++++++ perf-changelog.yaml | 8 ++ runners/launch_h200-dgxc-slurm.sh | 56 ++++++++- 5 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index a41936ea18..fde0a9f70b 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1801,7 +1801,11 @@ build_replay_cmd() { # aiperf's conv-aware routing emits nvext.session_control, a removed POC field # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400 # (they moved to router/routing_constraints/agent_context). Default stays on. - if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then + # New recipes instead set AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true + # to route by X-Dynamo-Session-ID header, which needs no routing CLI flag. + if [[ "${FRAMEWORK:-}" == dynamo-* \ + && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" \ + && "${AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID:-false}" != "true" ]]; then REPLAY_CMD+=" --use-dynamo-conv-aware-routing" # The upstream 300s affinity TTL is shorter than an overloaded # high-concurrency agentic request. Keep bindings alive across long diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..8fcaed4eb1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml @@ -0,0 +1,115 @@ +name: "agg-h200-tp8-mtp-kvoffload" + +# DeepSeek-V4-Pro AgentX aggregated TP8 serving on one 8xH200 node. The recipe is +# topology-invariant; configs/nvidia-master.yaml varies only concurrency to +# produce the latency/throughput Pareto curve. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp8" + +dynamo: + install: true + wheel: "1.3.0.dev20260718" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: true + # AgentX prompts can exceed NATS's 1 MiB default after JSON serialization. + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + aggregated_environment: + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_SIMULATE_ACC_LEN: "2.49" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: "1" + HF_HUB_CACHE: /hf_hub_cache + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + stream-interval: 50 + watchdog-timeout: 1000000 + mem-fraction-static: 0.88 + chunked-prefill-size: 4096 + moe-runner-backend: marlin + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs: 32 + + scheduler-recv-interval: 30 + tp-size: 8 + dp-size: 1 + ep-size: 1 + + enable-hierarchical-cache: true + hicache-ratio: 6 + hicache-write-policy: write_through + hicache-io-backend: kernel + + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +srun_options: + # The custom benchmark installs its isolated AIPerf environment at runtime. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated serving uses one TP8 worker for both prefill and decode. + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36ae816809..9411a950ad 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1695,6 +1695,41 @@ dsv4-fp8-h200-vllm-agentic: - search-space: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } +# DeepSeek-V4-Pro AgentX on one aggregated TP8 H200 worker. Keep the serving +# topology fixed and sweep only concurrency to produce the Pareto curve. +dsv4-fp8-h200-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:h200-dgxc + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.3.0.dev20260718" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [1, 2, 4, 8, 16] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml" + # The aggregated worker serves prefill and decode on the same GPUs. + # Keep decode at zero to avoid double-counting the TP8 allocation. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + # MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image, # runner pool, search space) and adds EAGLE speculative decoding via # --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0524d0c0c9..bdb06098ac 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5116,3 +5116,11 @@ description: - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - dsv4-fp8-h200-dynamo-sglang-agentic-agg + description: + - "Add one H200 TP8 aggregated DeepSeek-V4-Pro AgentX recipe with MTP and HiCache." + - "Generate Pareto points by sweeping concurrency [1, 2, 4, 8, 16] while keeping the serving topology fixed." + - "Use Dynamo header-based session affinity and pin AIPerf to the merged PR #17 implementation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 00c6cc4977..a7fc7279ff 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -4,6 +4,8 @@ set -eo pipefail # System-specific configuration for H200 DGXC Slurm cluster SLURM_PARTITION="main" SLURM_ACCOUNT="sa-shared" +HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/models/gharunners/hf-hub-cache}" +AIPERF_MMAP_CACHE_HOST_PATH="${AIPERF_MMAP_CACHE_HOST_PATH:-/home/sa-shared/gharunners/ai-perf-cache}" set -x @@ -13,7 +15,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H200 cluster. if [[ $FRAMEWORK == "dynamo-sglang" ]]; then - if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then + if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp8" ]]; then + # The shared HF cache already contains the H200 FP8 checkpoint; + # default to that local path (overridable via DSV4_MODEL_PATH) so + # srtctl preflight finds the directory instead of trying to pull the + # hf: model ID, which fails on the compute node ("path is + # unavailable. Pull or register the model yourself"). + export MODEL_PATH="${DSV4_MODEL_PATH:-${HF_HUB_CACHE_MOUNT}/DeepSeek-V4-Pro}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" + elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" else @@ -42,7 +52,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then + if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then + # Overlay the single H200 aggregated recipe on the upstream release + # that provides custom benchmarks, Dynamo wheels, and affinity config. + git clone --branch v1.0.10 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + mkdir -p recipes/sglang/deepseek-v4/agentic + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml" \ + recipes/sglang/deepseek-v4/agentic/ + elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" else @@ -67,6 +85,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Configs available at: $SRT_REPO_DIR/" # Map container images to local squash files based on framework + NGINX_IMAGE="nginx:1.27.4" NGINX_SQUASH_FILE="/data/containers/nginx+1.27.4.sqsh" if [[ $FRAMEWORK == "dynamo-sglang" ]]; then @@ -79,6 +98,31 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" fi + if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then + # This path intentionally tracks a recent public SGLang image instead + # of assuming the H200 cluster has already cached it. Import on a + # compute node and serialize concurrent jobs that share the cache. + import_squash() { + local squash="$1" image="$2" + local lock="${squash}.lock" + srun -N 1 --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" \ + --exclusive --time=180 bash -c " + exec 9>\"$lock\" + flock -w 600 9 || { echo 'Failed to acquire lock for $squash' >&2; exit 1; } + if unsquashfs -l \"$squash\" > /dev/null 2>&1; then + echo 'Squash file already exists and is valid, skipping import: $squash' + else + rm -f \"$squash\" + enroot import -o \"$squash\" docker://$image + unsquashfs -l \"$squash\" > /dev/null 2>&1 + fi + " + } + + import_squash "$SQUASH_FILE" "$IMAGE" + import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" + fi + export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -98,6 +142,11 @@ gpus_per_node: 8 network_interface: "" # Path to srtctl repo root (where the configs live) srtctl_root: "${SRTCTL_ROOT}" +# Persistent AgentX dataset and Hugging Face caches mounted into every +# server and benchmark container. +default_mounts: + "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" + "${HF_HUB_CACHE_MOUNT}": "/hf_hub_cache" # Model path aliases model_paths: "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}" @@ -271,9 +320,6 @@ EOF find . -name '.nfs*' -delete 2>/dev/null || true else - - HF_HUB_CACHE_MOUNT="/models/gharunners/hf-hub-cache" - AIPERF_MMAP_CACHE_HOST_PATH="/home/sa-shared/gharunners/ai-perf-cache" SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" # Convert pyxis image format (nvcr.io#path) to docker format (nvcr.io/path) for enroot import From 193a0c96ec2808652971ac6709985d02e5b14dbb Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Mon, 27 Jul 2026 15:30:35 -0700 Subject: [PATCH 2/3] perf-changelog: set dsv4 H200 agentic pr-link to #2364 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bdb06098ac..3025c11a60 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,4 +5123,4 @@ - "Add one H200 TP8 aggregated DeepSeek-V4-Pro AgentX recipe with MTP and HiCache." - "Generate Pareto points by sweeping concurrency [1, 2, 4, 8, 16] while keeping the serving topology fixed." - "Use Dynamo header-based session affinity and pin AIPerf to the merged PR #17 implementation." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2364 From 61a9767f6c29feecf3437309bcdabfce48efef1a Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 29 Jul 2026 17:49:16 -0700 Subject: [PATCH 3/3] h200 dsv4 agentic: drop import_squash, use pre-staged cluster sqsh The SGLang image is already staged on the H200 cluster at /data/containers/*.sqsh, like the dsr1 multinode path which imports nothing. Drop the import_squash helper and its now-unused NGINX_IMAGE; the multinode path maps SQUASH_FILE into srtslurm.yaml directly. --- runners/launch_h200-dgxc-slurm.sh | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index a7fc7279ff..6feadd9747 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -85,7 +85,6 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Configs available at: $SRT_REPO_DIR/" # Map container images to local squash files based on framework - NGINX_IMAGE="nginx:1.27.4" NGINX_SQUASH_FILE="/data/containers/nginx+1.27.4.sqsh" if [[ $FRAMEWORK == "dynamo-sglang" ]]; then @@ -98,31 +97,6 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" fi - if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # This path intentionally tracks a recent public SGLang image instead - # of assuming the H200 cluster has already cached it. Import on a - # compute node and serialize concurrent jobs that share the cache. - import_squash() { - local squash="$1" image="$2" - local lock="${squash}.lock" - srun -N 1 --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" \ - --exclusive --time=180 bash -c " - exec 9>\"$lock\" - flock -w 600 9 || { echo 'Failed to acquire lock for $squash' >&2; exit 1; } - if unsquashfs -l \"$squash\" > /dev/null 2>&1; then - echo 'Squash file already exists and is valid, skipping import: $squash' - else - rm -f \"$squash\" - enroot import -o \"$squash\" docker://$image - unsquashfs -l \"$squash\" > /dev/null 2>&1 - fi - " - } - - import_squash "$SQUASH_FILE" "$IMAGE" - import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" - fi - export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}"