From 59228fe2b63bd37a384ee205626824482ec26b3d Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:15:13 -0400 Subject: [PATCH 1/3] feat(qwen3.5-h200-agentic-mtp): add first Qwen3.5 H200 AgentX recipe, MTP with golden-AL simulated acceptance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add qwen3.5-fp8-h200-sglang-agentic-mtp: the first Qwen3.5 agentic-coding recipe on H200, shipped spec-decode-only per the AgentX policy in MODELS.md. SGLang EAGLE MTP (3 steps, eagle-topk 1, 4 draft tokens) with simulated acceptance pinned to the committed golden AL 3.39 (qwen3.5_mtp.yaml, thinking_on, K=3). Replay path follows the proven H100 Qwen3.5 AgentX recipe with --mem-fraction-static 0.8 for H200's 141 GB. EVAL_ONLY keeps real verification. 中文：新增 qwen3.5-fp8-h200-sglang-agentic-mtp —— H200 上首个 Qwen3.5 智能体编码 配方，按 MODELS.md 的 AgentX 规则仅发布投机解码版本。使用 SGLang EAGLE MTP（3 步、 eagle-topk 1、4 个草稿 token），模拟接受长度固定为黄金 AL 3.39（qwen3.5_mtp.yaml， thinking_on，K=3）。回放路径沿用已验证的 H100 Qwen3.5 AgentX 配方，并针对 H200 的 141 GB 显存将 --mem-fraction-static 调整为 0.8。EVAL_ONLY 保留真实验证。 Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic/qwen3.5_fp8_h200_mtp.sh | 183 ++++++++++++++++++ configs/nvidia-master.yaml | 25 +++ perf-changelog.yaml | 11 ++ 3 files changed, 219 insertions(+) create mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh new file mode 100755 index 0000000000..62982a3974 --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Qwen3.5 FP8 on H200 using SGLang with MTP +# speculative decoding. First Qwen3.5 AgentX recipe on H200; it is spec-decode +# only, per the AgentX policy that new agentic arms ship with speculative +# decoding enabled rather than as an STP/MTP A/B (MODELS.md). +# +# Structure follows the proven agentic/qwen3.5_fp8_h100.sh replay path +# (HiCache host-DRAM offload, the multi_tokenizer cached_tokens_details patch, +# aiperf-driven trace replay). H200's 141 GB HBM3e is roomier than H100's 80 GB, +# so --mem-fraction-static is 0.8 rather than 0.75, matching +# fixed_seq_len/qwen3.5_fp8_h200_mtp.sh. Attention backend stays flashinfer +# (sm_90); the trtllm_mha path is Blackwell-only. +# +# Speculative decoding mirrors fixed_seq_len/qwen3.5_fp8_h100_mtp.sh: +# SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, 3 steps, eagle-topk +# 1 and 4 draft tokens, i.e. 3 speculative tokens per verification step. +# +# Throughput runs pin acceptance to the committed golden AL through SGLang's +# simulated-acceptance path; the EVAL_ONLY accuracy run leaves it off and keeps +# real verification. See the SGLANG_SIMULATE_ACC_* block. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE + +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +nvidia-smi + +# ---- Resolve traces and install deps ---------------------------------------- +# Keep the 256k-capped with-subagents corpus the H100 Qwen3.5 AgentX recipe +# uses (470 traces, max in+out <= 256k). The unfiltered corpus has requests up +# to ~1M proxy tokens that the server would reject; H200's extra HBM raises the +# context ceiling but not past 256k for this model at TP8. +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k + +resolve_trace_source +install_agentic_deps + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +CACHE_ARGS=() +if require_agentic_kv_offload_backend hicache; then + # HiCache extends RadixAttention, so do not pass --disable-radix-cache. + # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank. + REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" + if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 + exit 1 + fi + TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" + HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) + HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" + if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then + echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 + exit 1 + fi + if [ "$HICACHE_SIZE_GB" -lt 1 ]; then + echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 + exit 1 + fi + echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" + CACHE_ARGS=( + --page-size 64 + --enable-hierarchical-cache + --hicache-size "$HICACHE_SIZE_GB" + --hicache-io-backend kernel + --hicache-mem-layout page_first + --hicache-write-policy "$HICACHE_WRITE_POLICY" + ) +fi + +echo "Starting SGLang server..." +export PYTHONNOUSERSITE=1 +export SGLANG_ENABLE_SPEC_V2=1 + +# 3 speculative tokens per step (num-steps 3, eagle-topk 1, 4 draft tokens), +# the same MTP shape as the fixed-seq-len Qwen3.5 recipes. +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +# AgentX pins acceptance to the committed golden AL so submissions are compared +# on system performance at a fixed acceptance target rather than on draft-head +# quality (golden_al_distribution/README.md). 3.39 is the Qwen3.5 MTP curve at +# num_speculative_tokens=3, thinking_on (golden_al_distribution/qwen3.5_mtp.yaml) +# -- the same value the GB300 Qwen3.5 AgentX srt-slurm recipes pin. +# SGLANG_SIMULATE_ACC_TOKEN_MODE landed in SGLang v0.5.16, which is why this +# recipe pins that image rather than the non-MTP agentic sibling's v0.5.12. +# +# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens +# regardless of the target logits, so generated text is wrong and the eval would +# score ~0. +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=3.39 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +SGLANG_MULTI_TOKENIZER=/sgl-workspace/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py +if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/p' "$SGLANG_MULTI_TOKENIZER" \ + | grep -q 'cached_tokens_details=_extract_field_by_index'; then + sed -i '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/ { + /cached_tokens=_extract_field_by_index(output, "cached_tokens", i),/a\ + cached_tokens_details=_extract_field_by_index(\ + output, "cached_tokens_details", i\ + ), + }' "$SGLANG_MULTI_TOKENIZER" +fi + +{ set +x; } 2>/dev/null +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path="$MODEL_PATH" --served-model-name="$MODEL" + --host=0.0.0.0 + --port="$PORT" + --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" + --trust-remote-code + --tensor-parallel-size="$TP" + --data-parallel-size=1 + --expert-parallel-size="$EP_SIZE" + --quantization fp8 + --kv-cache-dtype fp8_e4m3 + --mamba-ssm-dtype bfloat16 + --attention-backend flashinfer + --enable-flashinfer-allreduce-fusion + # --cuda-graph-max-bs "$CONC" + # --max-running-requests "$CONC" + # --max-prefill-tokens 8192 + # --chunked-prefill-size 8192 + --mem-fraction-static 0.8 + --stream-interval 50 + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" + --tokenizer-worker-num 6 + --tokenizer-path "$MODEL" + --enable-metrics + "${SPEC_ARGS[@]}" + "${CACHE_ARGS[@]}" +) +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8ee037c8c2..b619627f79 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7242,6 +7242,31 @@ qwen3.5-fp8-h100-sglang-agentic: - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] } +# First Qwen3.5 AgentX recipe on H200, shipped spec-decode-only per the AgentX +# policy that new agentic arms enable speculative decoding rather than running a +# separate STP baseline (MODELS.md). SGLang EAGLE MTP (num-steps 3, eagle-topk 1, +# 4 draft tokens = 3 speculative tokens) with simulated acceptance pinned to the +# golden AL 3.39 (golden_al_distribution/qwen3.5_mtp.yaml, thinking_on, K=3). +# Image is lmsysorg/sglang:v0.5.16-cu130: SGLANG_SIMULATE_ACC_TOKEN_MODE only +# exists from v0.5.16, and v0.5.14 is what the fixed-seq-len H200 MTP entry runs. +# Search space is the H100 AgentX shape widened for H200's 141 GB: the +# fixed-seq-len H200 MTP entry already runs conc 4-128 at TP8/EP8 versus H100's +# 4-32, so the GPU-resident arm extends to 24 and HiCache to 48. +qwen3.5-fp8-h200-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.16-cu130 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:h200-dgxc + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20, 24] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 32, 40, 48] } + minimaxm3-fp4-b200-dynamo-vllm: image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: nvidia/MiniMax-M3-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1736ae420..6f4ef8ffe8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5311,3 +5311,14 @@ - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs." - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2418 + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-mtp + description: + - "Add the first Qwen3.5 AgentX (agentic-coding) recipe on H200: Qwen3.5-397B-A17B FP8 with SGLang, spec-decoding=mtp only, routed to benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B, because synthetic acceptance already makes spec-decode results comparable across submissions." + - "Speculative config mirrors fixed_seq_len/qwen3.5_fp8_h200_mtp.sh: SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4 -- 3 speculative tokens per verification step." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang, and the same AL the GB300 Qwen3.5 AgentX srt-slurm recipes pin. EVAL_ONLY runs leave simulated acceptance off and keep real verification." + - "Image lmsysorg/sglang:v0.5.16-cu130. SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16 (SIMULATE_ACC_LEN / SIMULATE_ACC_METHOD exist further back), so an older image would silently ignore the token-mode half of the AgentX contract; v0.5.14 is what the fixed-seq-len H200 Qwen3.5 entries already run." + - "Serve shape follows the proven H100 Qwen3.5 AgentX replay path (SGLang HiCache host-DRAM offload with page-size 64 / kernel IO / page_first layout, flashinfer attention, --quantization fp8, --kv-cache-dtype fp8_e4m3, --mamba-ssm-dtype bfloat16, the multi_tokenizer cached_tokens_details patch, the 256k-capped with-subagents trace corpus) with --mem-fraction-static 0.8 instead of 0.75, matching the fixed-seq-len H200 recipe's use of H200's 141 GB HBM3e." + - "Search space is the H100 AgentX shape widened for H200 memory: TP8/EP8 GPU-resident 1-24 and HiCache 16-48. The fixed-seq-len H200 MTP entry already sweeps conc 4-128 at TP8/EP8 against H100's 4-32, so the agentic cliff sits meaningfully higher here. TP8-only, as on every single-node Qwen3.5 H100/H200 entry." + pr-link: TBD From 3528d954261cfeac38be199dee978b8b6b559fb4 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:15:32 -0400 Subject: [PATCH 2/3] chore: fill perf-changelog pr-link for #2428 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：补充 perf-changelog 中 #2428 的 pr-link。 Co-Authored-By: Claude Opus 5 (1M context) --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6f4ef8ffe8..1d319b2cc1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5321,4 +5321,4 @@ - "Image lmsysorg/sglang:v0.5.16-cu130. SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16 (SIMULATE_ACC_LEN / SIMULATE_ACC_METHOD exist further back), so an older image would silently ignore the token-mode half of the AgentX contract; v0.5.14 is what the fixed-seq-len H200 Qwen3.5 entries already run." - "Serve shape follows the proven H100 Qwen3.5 AgentX replay path (SGLang HiCache host-DRAM offload with page-size 64 / kernel IO / page_first layout, flashinfer attention, --quantization fp8, --kv-cache-dtype fp8_e4m3, --mamba-ssm-dtype bfloat16, the multi_tokenizer cached_tokens_details patch, the 256k-capped with-subagents trace corpus) with --mem-fraction-static 0.8 instead of 0.75, matching the fixed-seq-len H200 recipe's use of H200's 141 GB HBM3e." - "Search space is the H100 AgentX shape widened for H200 memory: TP8/EP8 GPU-resident 1-24 and HiCache 16-48. The fixed-seq-len H200 MTP entry already sweeps conc 4-128 at TP8/EP8 against H100's 4-32, so the agentic cliff sits meaningfully higher here. TP8-only, as on every single-node Qwen3.5 H100/H200 entry." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2428 From 02b418c1e2d95933283ecde8eb32e2ff16e219b3 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:42:31 -0400 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20AgentX=20MTP=20conc=20grid=20?= =?UTF-8?q?=E2=80=94=20steps=20>=3D2,=20hard=20stop=20at=20conc=2016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the concurrency arms onto the AgentX MTP grid: GPU-resident [1,4,8,12,16], HiCache [4,8,12,16]. Every step is at least 2 apart and every arm stops hard at conc 16, instead of the single-step / high-conc sampling carried over from the non-MTP agentic shapes. 中文：将并发分支调整为 AgentX MTP 并发网格：GPU 常驻 [1,4,8,12,16]，HiCache [4,8,12,16]。相邻并发间隔至少为 2，且一律止于 16，不再沿用非 MTP 智能体配方的 逐 1 递增与高并发采样。 Co-Authored-By: Claude Opus 5 (1M context) --- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b619627f79..41942dc8fe 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7264,8 +7264,8 @@ qwen3.5-fp8-h200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20, 24] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 32, 40, 48] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] } minimaxm3-fp4-b200-dynamo-vllm: image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1d319b2cc1..cba5858913 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5320,5 +5320,5 @@ - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang, and the same AL the GB300 Qwen3.5 AgentX srt-slurm recipes pin. EVAL_ONLY runs leave simulated acceptance off and keep real verification." - "Image lmsysorg/sglang:v0.5.16-cu130. SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16 (SIMULATE_ACC_LEN / SIMULATE_ACC_METHOD exist further back), so an older image would silently ignore the token-mode half of the AgentX contract; v0.5.14 is what the fixed-seq-len H200 Qwen3.5 entries already run." - "Serve shape follows the proven H100 Qwen3.5 AgentX replay path (SGLang HiCache host-DRAM offload with page-size 64 / kernel IO / page_first layout, flashinfer attention, --quantization fp8, --kv-cache-dtype fp8_e4m3, --mamba-ssm-dtype bfloat16, the multi_tokenizer cached_tokens_details patch, the 256k-capped with-subagents trace corpus) with --mem-fraction-static 0.8 instead of 0.75, matching the fixed-seq-len H200 recipe's use of H200's 141 GB HBM3e." - - "Search space is the H100 AgentX shape widened for H200 memory: TP8/EP8 GPU-resident 1-24 and HiCache 16-48. The fixed-seq-len H200 MTP entry already sweeps conc 4-128 at TP8/EP8 against H100's 4-32, so the agentic cliff sits meaningfully higher here. TP8-only, as on every single-node Qwen3.5 H100/H200 entry." + - "Search space now runs on the AgentX MTP concurrency grid: the GPU-resident row [1, 4, 8, 12, 16] and the HiCache row [4, 8, 12, 16]. Every step is at least 2 concurrency apart and every arm stops hard at conc 16 -- single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and past conc 16 these SKUs are into the post-HBM-cliff thrashing regime that the non-MTP sweeps already characterized, which is not worth one GPU job per point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2428