From 9168f71cad572db811a71a5402e8b93155e1301a Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:17:04 -0400 Subject: [PATCH 1/4] feat(qwen3.5-mi325x-agentic-mtp): add first Qwen3.5 MI325X AgentX recipe, MTP with golden-AL simulated acceptance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add qwen3.5-fp8-mi325x-sglang-agentic-mtp: the first Qwen3.5 agentic-coding recipe on MI325X (gfx942), shipped spec-decode-only per the AgentX policy in MODELS.md. SGLang EAGLE MTP (3 steps, eagle-topk 1, 4 draft tokens) matching the fixed-seq-len MI325X recipe, with simulated acceptance pinned to the committed golden AL 3.39 (qwen3.5_mtp.yaml, thinking_on, K=3). Replay path follows the MI355X Qwen3.5 AgentX recipe (triton attention, page_size=1 direct/layer_first HiCache). EVAL_ONLY keeps real verification. 中文：新增 qwen3.5-fp8-mi325x-sglang-agentic-mtp —— MI325X（gfx942）上首个 Qwen3.5 智能体编码配方，按 MODELS.md 的 AgentX 规则仅发布投机解码版本。SGLang EAGLE MTP （3 步、eagle-topk 1、4 个草稿 token）与同 SKU 的固定序列长度配方一致；模拟接受 长度固定为黄金 AL 3.39（qwen3.5_mtp.yaml，thinking_on，K=3）。回放路径沿用 MI355X Qwen3.5 AgentX 配方（triton 注意力后端、page_size=1 的 direct/layer_first HiCache）。 EVAL_ONLY 保留真实验证。 Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic/qwen3.5_fp8_mi325x_mtp.sh | 188 ++++++++++++++++++ configs/amd-master.yaml | 26 +++ perf-changelog.yaml | 11 + 3 files changed, 225 insertions(+) create mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh new file mode 100755 index 0000000000..7815953361 --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Qwen3.5 FP8 on MI325X (gfx942) using SGLang +# with MTP speculative decoding. First Qwen3.5 AgentX recipe on MI325X; it is +# spec-decode only, per the AgentX policy that new agentic arms ship with +# speculative decoding enabled rather than as an STP/MTP A/B (MODELS.md). +# +# Structure follows the proven agentic/qwen3.5_fp8_mi355x_sglang.sh replay path, +# including its ROCm HiCache settings: Qwen3.5's hybrid Mamba path runs SGLang's +# no_buffer scheduler, which requires page_size=1, and the kernel/page_first +# HiCache transfer path faults on first prefill in that mode on ROCm, so the +# safer direct/layer_first copy path is kept. Attention backend is triton, as on +# the MI355X agentic recipe -- the fixed-seq-len gfx942 recipes use aiter, but +# the agentic replay path is validated on triton. --mem-fraction-static is 0.75, +# matching fixed_seq_len/qwen3.5_fp8_mi325x.sh; MI325X's 256 GB per GPU buys a +# larger KV pool at the same static fraction than MI300X's 192 GB. +# +# Speculative decoding mirrors fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh, the +# fixed-seq-len recipe on this same SKU: --speculative-algorithm EAGLE with 3 +# steps, eagle-topk 1 and 4 draft tokens, i.e. 3 speculative tokens per step. +# +# Throughput runs pin acceptance to the committed golden AL through SGLang's +# simulated-acceptance path; the EVAL_ONLY accuracy run leaves it off and keeps +# real verification. See the SGLANG_SIMULATE_ACC_* block. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE + +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +CACHE_ARGS=() +WARMUP_ARGS=() +CUDA_GRAPH_MAX_BS="$CONC" +if require_agentic_kv_offload_backend hicache; then + # Qwen3.5 allocates one KV and one Mamba host pool per TP rank. + REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" + if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 + exit 1 + fi + TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" + HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" + HICACHE_MAX_SIZE_GB_PER_RANK_POOL="${HICACHE_MAX_SIZE_GB_PER_RANK_POOL:-${HICACHE_MAX_SIZE_GB_PER_RANK:-180}}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + # Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler on MI355X, + # which requires page_size=1. The kernel/page_first HiCache transfer path + # faults on first prefill in this mode on ROCm, so keep the default on the + # safer direct/layer_first copy path. These remain env-overridable. + HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-layer_first}" + # SGLang --hicache-size is per rank per host pool, while the workflow + # input is a node-total DRAM budget. Divide by TP and the number of + # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning. + MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) + HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" + if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then + echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 + exit 1 + fi + if [ "$HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" ]; then + HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" + fi + if [ "$HICACHE_SIZE_GB" -lt 1 ]; then + echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 + exit 1 + fi + echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" + CACHE_ARGS=( + --page-size "$HICACHE_PAGE_SIZE" + --enable-hierarchical-cache + --hicache-size "$HICACHE_SIZE_GB" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + ) + # HiCache startup reaches API readiness, but SGLang's internal warmup + # request has timed out after 600s on this Qwen MI355X path. Let aiperf + # own benchmark traffic instead of blocking server readiness on it. + WARMUP_ARGS=(--skip-server-warmup) + # Keep request concurrency as the swept variable, but do not force HiCache + # runs to capture ROCm graphs at every high concurrency point. + HICACHE_CUDA_GRAPH_MAX_BS="${HICACHE_CUDA_GRAPH_MAX_BS:-16}" + if [ "$HICACHE_CUDA_GRAPH_MAX_BS" -lt "$CUDA_GRAPH_MAX_BS" ]; then + CUDA_GRAPH_MAX_BS="$HICACHE_CUDA_GRAPH_MAX_BS" + fi +fi + +echo "Starting SGLang server..." +export PYTHONNOUSERSITE=1 +export SGLANG_ENABLE_SPEC_V2=1 + +# 3 speculative tokens per step (num-steps 3, eagle-topk 1, 4 draft tokens), +# the same MTP shape as fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh. +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +# AgentX pins acceptance to the committed golden AL so submissions are compared +# on system performance at a fixed acceptance target rather than on draft-head +# quality (golden_al_distribution/README.md). 3.39 is the Qwen3.5 MTP curve at +# num_speculative_tokens=3, thinking_on (golden_al_distribution/qwen3.5_mtp.yaml) +# -- the same value the GB300 Qwen3.5 AgentX srt-slurm recipes pin. +# SGLANG_SIMULATE_ACC_TOKEN_MODE landed in SGLang v0.5.16, which is the image +# this recipe pins. +# +# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens +# regardless of the target logits, so generated text is wrong and the eval would +# score ~0. +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=3.39 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +{ set +x; } 2>/dev/null +SGLANG_CMD=( + python3 -m sglang.launch_server + --attention-backend triton + --model-path "$MODEL_PATH" --served-model-name "$MODEL" + --host=0.0.0.0 + --port "$PORT" + --tensor-parallel-size "$TP" + --ep-size "$EP_SIZE" + --trust-remote-code + --tokenizer-worker-num 6 + --enable-aiter-allreduce-fusion + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$CONC" + --max-prefill-tokens 32768 + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" + --mem-fraction-static 0.75 + --enable-metrics + "${SPEC_ARGS[@]}" + "${CACHE_ARGS[@]}" + "${WARMUP_ARGS[@]}" +) +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b15ae5bccd..0a27741161 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1495,6 +1495,32 @@ qwen3.5-fp8-mi325x-sglang-mtp: search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } +# First Qwen3.5 AgentX (agentic-coding) recipe on MI325X, shipped spec-decode-only +# per the AgentX policy that new agentic arms enable speculative decoding rather +# than running a separate STP baseline (MODELS.md). Same SGLang EAGLE MTP shape as +# the fixed-seq-len MI325X entry above (num-steps 3, eagle-topk 1, 4 draft tokens +# = 3 speculative tokens), with simulated acceptance pinned to the golden AL 3.39 +# (golden_al_distribution/qwen3.5_mtp.yaml, thinking_on, K=3). Image moves to +# lmsysorg/sglang:v0.5.16-rocm720-mi30x because SGLANG_SIMULATE_ACC_TOKEN_MODE +# only exists from v0.5.16. Replay path follows the MI355X Qwen3.5 AgentX recipe +# (triton attention, HiCache on the page_size=1 direct/layer_first copy path +# required by Qwen3.5's hybrid Mamba no_buffer scheduler on ROCm). Conc bands run +# past the MI300X entry's because 256 GB per GPU holds a larger KV pool. +qwen3.5-fp8-mi325x-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.16-rocm720-mi30x + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi325x-amds + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 24] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 24, 32, 48] } + qwen3.5-fp8-mi355x-sglang-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260521 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1736ae420..6aff410963 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5311,3 +5311,14 @@ - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs." - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2418 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang-agentic-mtp + description: + - "Add the first Qwen3.5 AgentX (agentic-coding) recipe on MI325X: Qwen3.5-397B-A17B FP8 with SGLang, spec-decoding=mtp only, routed to benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B." + - "Speculative config mirrors fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh, the fixed-seq-len recipe on this same SKU: SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4 -- 3 speculative tokens per verification step." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang. EVAL_ONLY runs leave simulated acceptance off and keep real verification." + - "Image lmsysorg/sglang:v0.5.16-rocm720-mi30x (tag verified on Docker Hub), up from the fixed-seq-len MI325X entry's v0.5.12-rocm720-mi30x: SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16, so an older image would silently ignore the token-mode half of the AgentX contract." + - "Replay path follows the MI355X Qwen3.5 AgentX recipe rather than the fixed-seq-len gfx942 one: --attention-backend triton (the fixed-seq-len ROCm recipes use aiter, but the agentic corpus is validated on triton), HiCache host-DRAM offload forced onto the page_size=1 direct/layer_first copy path because Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler and the kernel/page_first transfer path faults on first prefill on ROCm, --skip-server-warmup so aiperf owns benchmark traffic, and the HiCache cudagraph-batch cap. --mem-fraction-static stays 0.75 as on fixed_seq_len/qwen3.5_fp8_mi325x.sh." + - "Search space is a TP8-only bring-up shape one band wider than the MI300X sibling -- GPU-resident conc 1-24 and HiCache conc 16-48 -- because MI325X's 256 GB per GPU holds a larger KV pool at the same static fraction. Widen further after the first sweep locates the cliff." + pr-link: TBD From 56cb999dd3e83e8390e603e8403c677379f09f94 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:17:19 -0400 Subject: [PATCH 2/4] chore: fill perf-changelog pr-link for #2430 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：补充 perf-changelog 中 #2430 的 pr-link。 Co-Authored-By: Claude Opus 5 (1M context) --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6aff410963..0df1afd80e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5321,4 +5321,4 @@ - "Image lmsysorg/sglang:v0.5.16-rocm720-mi30x (tag verified on Docker Hub), up from the fixed-seq-len MI325X entry's v0.5.12-rocm720-mi30x: SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16, so an older image would silently ignore the token-mode half of the AgentX contract." - "Replay path follows the MI355X Qwen3.5 AgentX recipe rather than the fixed-seq-len gfx942 one: --attention-backend triton (the fixed-seq-len ROCm recipes use aiter, but the agentic corpus is validated on triton), HiCache host-DRAM offload forced onto the page_size=1 direct/layer_first copy path because Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler and the kernel/page_first transfer path faults on first prefill on ROCm, --skip-server-warmup so aiperf owns benchmark traffic, and the HiCache cudagraph-batch cap. --mem-fraction-static stays 0.75 as on fixed_seq_len/qwen3.5_fp8_mi325x.sh." - "Search space is a TP8-only bring-up shape one band wider than the MI300X sibling -- GPU-resident conc 1-24 and HiCache conc 16-48 -- because MI325X's 256 GB per GPU holds a larger KV pool at the same static fraction. Widen further after the first sweep locates the cliff." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2430 From 29e2f86e590288a536cd6537296857a153fc5d2c Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:42:34 -0400 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20AgentX=20MTP=20conc=20grid=20?= =?UTF-8?q?=E2=80=94=20steps=20>=3D2,=20hard=20stop=20at=20conc=2016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the concurrency arms onto the AgentX MTP grid: GPU-resident [1,4,8,12,16], HiCache [4,8,12,16]. Every step is at least 2 apart and every arm stops hard at conc 16, instead of the single-step / high-conc sampling carried over from the non-MTP agentic shapes. 中文：将并发分支调整为 AgentX MTP 并发网格：GPU 常驻 [1,4,8,12,16]，HiCache [4,8,12,16]。相邻并发间隔至少为 2，且一律止于 16，不再沿用非 MTP 智能体配方的 逐 1 递增与高并发采样。 Co-Authored-By: Claude Opus 5 (1M context) --- configs/amd-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0a27741161..491c96b561 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1518,8 +1518,8 @@ qwen3.5-fp8-mi325x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 24] } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 24, 32, 48] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] } qwen3.5-fp8-mi355x-sglang-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260521 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0df1afd80e..1835f53e34 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5320,5 +5320,5 @@ - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang. EVAL_ONLY runs leave simulated acceptance off and keep real verification." - "Image lmsysorg/sglang:v0.5.16-rocm720-mi30x (tag verified on Docker Hub), up from the fixed-seq-len MI325X entry's v0.5.12-rocm720-mi30x: SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16, so an older image would silently ignore the token-mode half of the AgentX contract." - "Replay path follows the MI355X Qwen3.5 AgentX recipe rather than the fixed-seq-len gfx942 one: --attention-backend triton (the fixed-seq-len ROCm recipes use aiter, but the agentic corpus is validated on triton), HiCache host-DRAM offload forced onto the page_size=1 direct/layer_first copy path because Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler and the kernel/page_first transfer path faults on first prefill on ROCm, --skip-server-warmup so aiperf owns benchmark traffic, and the HiCache cudagraph-batch cap. --mem-fraction-static stays 0.75 as on fixed_seq_len/qwen3.5_fp8_mi325x.sh." - - "Search space is a TP8-only bring-up shape one band wider than the MI300X sibling -- GPU-resident conc 1-24 and HiCache conc 16-48 -- because MI325X's 256 GB per GPU holds a larger KV pool at the same static fraction. Widen further after the first sweep locates the cliff." + - "Search space now runs on the AgentX MTP concurrency grid: the GPU-resident row [1, 4, 8, 12, 16] and the HiCache row [4, 8, 12, 16]. Every step is at least 2 concurrency apart and every arm stops hard at conc 16 -- single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and past conc 16 these SKUs are into the post-HBM-cliff thrashing regime that the non-MTP sweeps already characterized, which is not worth one GPU job per point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2430 From e9c57212b9b3433c569c4b0214ca860106bb4cf0 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:23:27 -0400 Subject: [PATCH 4/4] =?UTF-8?q?fix(qwen3.5-mi325x-agentic-mtp):=20use=20th?= =?UTF-8?q?e=20sglang-rocm=20image=20=E2=80=94=20the=20plain=20mi30x=20tag?= =?UTF-8?q?=20ships=20Python=203.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MI300X sibling died ~30s in, before serving: lmsysorg/sglang:v0.5.16-rocm720-mi30x ships Python 3.10.12 in /opt/venv, install_agentic_deps builds the AgentX venv from $(command -v python3), and uv cannot resolve aiperf (requires Python>=3.11,<3.14). This recipe pinned the same image, so it would fail identically once its jobs started. Switch to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730: same sglang 0.5.16 (SGLANG_SIMULATE_ACC_TOKEN_MODE still honored), same gfx942 arch, and the dated sglang-rocm family ships CPython 3.11.15 -- verified from a green MI355X AgentX job. 中文：MI300X 版本在约 30 秒后、尚未启动服务前即失败：镜像 lmsysorg/sglang:v0.5.16-rocm720-mi30x 的 /opt/venv 为 Python 3.10.12，而 install_agentic_deps 依据 $(command -v python3) 创建 AgentX 虚拟环境，uv 无法解析要求 Python>=3.11 的 aiperf。本配方此前固定了同一镜像，作业一旦启动必然同样失败。现改用 lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730：sglang 仍为 0.5.16、架构仍为 gfx942，且该系列自带 CPython 3.11.15 —— 已由成功的 MI355X AgentX 作业验证。 Co-Authored-By: Claude Opus 5 (1M context) --- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 491c96b561..b4c9a2f853 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1507,7 +1507,7 @@ qwen3.5-fp8-mi325x-sglang-mtp: # required by Qwen3.5's hybrid Mamba no_buffer scheduler on ROCm). Conc bands run # past the MI300X entry's because 256 GB per GPU holds a larger KV pool. qwen3.5-fp8-mi325x-sglang-agentic-mtp: - image: lmsysorg/sglang:v0.5.16-rocm720-mi30x + image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: cluster:mi325x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1835f53e34..073f6cb9dd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5318,7 +5318,7 @@ - "Add the first Qwen3.5 AgentX (agentic-coding) recipe on MI325X: Qwen3.5-397B-A17B FP8 with SGLang, spec-decoding=mtp only, routed to benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B." - "Speculative config mirrors fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh, the fixed-seq-len recipe on this same SKU: SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4 -- 3 speculative tokens per verification step." - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang. EVAL_ONLY runs leave simulated acceptance off and keep real verification." - - "Image lmsysorg/sglang:v0.5.16-rocm720-mi30x (tag verified on Docker Hub), up from the fixed-seq-len MI325X entry's v0.5.12-rocm720-mi30x: SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16, so an older image would silently ignore the token-mode half of the AgentX contract." + - "Image lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 (tag verified on Docker Hub). v0.5.16 is required because SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from that release. The dated sglang-rocm build is used rather than the plain lmsysorg/sglang:v0.5.16-rocm720-mi30x tag: that image ships Python 3.10.12 in /opt/venv, and install_agentic_deps builds the AgentX venv from $(command -v python3), so uv cannot resolve aiperf (which requires Python >=3.11,<3.14) and the job dies ~30s in before serving. The lmsysorg/sglang-rocm dated family ships CPython 3.11.15 and is what the working MI355X AgentX recipes run." - "Replay path follows the MI355X Qwen3.5 AgentX recipe rather than the fixed-seq-len gfx942 one: --attention-backend triton (the fixed-seq-len ROCm recipes use aiter, but the agentic corpus is validated on triton), HiCache host-DRAM offload forced onto the page_size=1 direct/layer_first copy path because Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler and the kernel/page_first transfer path faults on first prefill on ROCm, --skip-server-warmup so aiperf owns benchmark traffic, and the HiCache cudagraph-batch cap. --mem-fraction-static stays 0.75 as on fixed_seq_len/qwen3.5_fp8_mi325x.sh." - "Search space now runs on the AgentX MTP concurrency grid: the GPU-resident row [1, 4, 8, 12, 16] and the HiCache row [4, 8, 12, 16]. Every step is at least 2 concurrency apart and every arm stops hard at conc 16 -- single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and past conc 16 these SKUs are into the post-HBM-cliff thrashing regime that the non-MTP sweeps already characterized, which is not worth one GPU job per point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2430