diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh new file mode 100755 index 0000000000..7815953361 --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Qwen3.5 FP8 on MI325X (gfx942) using SGLang +# with MTP speculative decoding. First Qwen3.5 AgentX recipe on MI325X; it is +# spec-decode only, per the AgentX policy that new agentic arms ship with +# speculative decoding enabled rather than as an STP/MTP A/B (MODELS.md). +# +# Structure follows the proven agentic/qwen3.5_fp8_mi355x_sglang.sh replay path, +# including its ROCm HiCache settings: Qwen3.5's hybrid Mamba path runs SGLang's +# no_buffer scheduler, which requires page_size=1, and the kernel/page_first +# HiCache transfer path faults on first prefill in that mode on ROCm, so the +# safer direct/layer_first copy path is kept. Attention backend is triton, as on +# the MI355X agentic recipe -- the fixed-seq-len gfx942 recipes use aiter, but +# the agentic replay path is validated on triton. --mem-fraction-static is 0.75, +# matching fixed_seq_len/qwen3.5_fp8_mi325x.sh; MI325X's 256 GB per GPU buys a +# larger KV pool at the same static fraction than MI300X's 192 GB. +# +# Speculative decoding mirrors fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh, the +# fixed-seq-len recipe on this same SKU: --speculative-algorithm EAGLE with 3 +# steps, eagle-topk 1 and 4 draft tokens, i.e. 3 speculative tokens per step. +# +# Throughput runs pin acceptance to the committed golden AL through SGLang's +# simulated-acceptance path; the EVAL_ONLY accuracy run leaves it off and keeps +# real verification. See the SGLANG_SIMULATE_ACC_* block. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE + +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +CACHE_ARGS=() +WARMUP_ARGS=() +CUDA_GRAPH_MAX_BS="$CONC" +if require_agentic_kv_offload_backend hicache; then + # Qwen3.5 allocates one KV and one Mamba host pool per TP rank. + REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" + if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 + exit 1 + fi + TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" + HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" + HICACHE_MAX_SIZE_GB_PER_RANK_POOL="${HICACHE_MAX_SIZE_GB_PER_RANK_POOL:-${HICACHE_MAX_SIZE_GB_PER_RANK:-180}}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + # Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler on MI355X, + # which requires page_size=1. The kernel/page_first HiCache transfer path + # faults on first prefill in this mode on ROCm, so keep the default on the + # safer direct/layer_first copy path. These remain env-overridable. + HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-layer_first}" + # SGLang --hicache-size is per rank per host pool, while the workflow + # input is a node-total DRAM budget. Divide by TP and the number of + # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning. + MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) + HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" + if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then + echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 + exit 1 + fi + if [ "$HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" ]; then + HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" + fi + if [ "$HICACHE_SIZE_GB" -lt 1 ]; then + echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 + exit 1 + fi + echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" + CACHE_ARGS=( + --page-size "$HICACHE_PAGE_SIZE" + --enable-hierarchical-cache + --hicache-size "$HICACHE_SIZE_GB" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + ) + # HiCache startup reaches API readiness, but SGLang's internal warmup + # request has timed out after 600s on this Qwen MI355X path. Let aiperf + # own benchmark traffic instead of blocking server readiness on it. + WARMUP_ARGS=(--skip-server-warmup) + # Keep request concurrency as the swept variable, but do not force HiCache + # runs to capture ROCm graphs at every high concurrency point. + HICACHE_CUDA_GRAPH_MAX_BS="${HICACHE_CUDA_GRAPH_MAX_BS:-16}" + if [ "$HICACHE_CUDA_GRAPH_MAX_BS" -lt "$CUDA_GRAPH_MAX_BS" ]; then + CUDA_GRAPH_MAX_BS="$HICACHE_CUDA_GRAPH_MAX_BS" + fi +fi + +echo "Starting SGLang server..." +export PYTHONNOUSERSITE=1 +export SGLANG_ENABLE_SPEC_V2=1 + +# 3 speculative tokens per step (num-steps 3, eagle-topk 1, 4 draft tokens), +# the same MTP shape as fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh. +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +# AgentX pins acceptance to the committed golden AL so submissions are compared +# on system performance at a fixed acceptance target rather than on draft-head +# quality (golden_al_distribution/README.md). 3.39 is the Qwen3.5 MTP curve at +# num_speculative_tokens=3, thinking_on (golden_al_distribution/qwen3.5_mtp.yaml) +# -- the same value the GB300 Qwen3.5 AgentX srt-slurm recipes pin. +# SGLANG_SIMULATE_ACC_TOKEN_MODE landed in SGLang v0.5.16, which is the image +# this recipe pins. +# +# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens +# regardless of the target logits, so generated text is wrong and the eval would +# score ~0. +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=3.39 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +{ set +x; } 2>/dev/null +SGLANG_CMD=( + python3 -m sglang.launch_server + --attention-backend triton + --model-path "$MODEL_PATH" --served-model-name "$MODEL" + --host=0.0.0.0 + --port "$PORT" + --tensor-parallel-size "$TP" + --ep-size "$EP_SIZE" + --trust-remote-code + --tokenizer-worker-num 6 + --enable-aiter-allreduce-fusion + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$CONC" + --max-prefill-tokens 32768 + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" + --mem-fraction-static 0.75 + --enable-metrics + "${SPEC_ARGS[@]}" + "${CACHE_ARGS[@]}" + "${WARMUP_ARGS[@]}" +) +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b15ae5bccd..b4c9a2f853 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1495,6 +1495,32 @@ qwen3.5-fp8-mi325x-sglang-mtp: search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } +# First Qwen3.5 AgentX (agentic-coding) recipe on MI325X, shipped spec-decode-only +# per the AgentX policy that new agentic arms enable speculative decoding rather +# than running a separate STP baseline (MODELS.md). Same SGLang EAGLE MTP shape as +# the fixed-seq-len MI325X entry above (num-steps 3, eagle-topk 1, 4 draft tokens +# = 3 speculative tokens), with simulated acceptance pinned to the golden AL 3.39 +# (golden_al_distribution/qwen3.5_mtp.yaml, thinking_on, K=3). Image moves to +# lmsysorg/sglang:v0.5.16-rocm720-mi30x because SGLANG_SIMULATE_ACC_TOKEN_MODE +# only exists from v0.5.16. Replay path follows the MI355X Qwen3.5 AgentX recipe +# (triton attention, HiCache on the page_size=1 direct/layer_first copy path +# required by Qwen3.5's hybrid Mamba no_buffer scheduler on ROCm). Conc bands run +# past the MI300X entry's because 256 GB per GPU holds a larger KV pool. +qwen3.5-fp8-mi325x-sglang-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi325x-amds + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] } + qwen3.5-fp8-mi355x-sglang-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260521 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4c22ed9198..94ed829949 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5347,3 +5347,14 @@ - "Add seven checked-in srt-slurm recipes selected one-to-one by the 8K/1K prefill/decode topology and concurrency entries." - "Image: vllm/vllm-openai:v0.25.1" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2360 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang-agentic-mtp + description: + - "Add the first Qwen3.5 AgentX (agentic-coding) recipe on MI325X: Qwen3.5-397B-A17B FP8 with SGLang, spec-decoding=mtp only, routed to benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B." + - "Speculative config mirrors fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh, the fixed-seq-len recipe on this same SKU: SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4 -- 3 speculative tokens per verification step." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang. EVAL_ONLY runs leave simulated acceptance off and keep real verification." + - "Image lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 (tag verified on Docker Hub). v0.5.16 is required because SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from that release. The dated sglang-rocm build is used rather than the plain lmsysorg/sglang:v0.5.16-rocm720-mi30x tag: that image ships Python 3.10.12 in /opt/venv, and install_agentic_deps builds the AgentX venv from $(command -v python3), so uv cannot resolve aiperf (which requires Python >=3.11,<3.14) and the job dies ~30s in before serving. The lmsysorg/sglang-rocm dated family ships CPython 3.11.15 and is what the working MI355X AgentX recipes run." + - "Replay path follows the MI355X Qwen3.5 AgentX recipe rather than the fixed-seq-len gfx942 one: --attention-backend triton (the fixed-seq-len ROCm recipes use aiter, but the agentic corpus is validated on triton), HiCache host-DRAM offload forced onto the page_size=1 direct/layer_first copy path because Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler and the kernel/page_first transfer path faults on first prefill on ROCm, --skip-server-warmup so aiperf owns benchmark traffic, and the HiCache cudagraph-batch cap. --mem-fraction-static stays 0.75 as on fixed_seq_len/qwen3.5_fp8_mi325x.sh." + - "Search space now runs on the AgentX MTP concurrency grid: the GPU-resident row [1, 4, 8, 12, 16] and the HiCache row [4, 8, 12, 16]. Every step is at least 2 concurrency apart and every arm stops hard at conc 16 -- single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and past conc 16 these SKUs are into the post-HBM-cliff thrashing regime that the non-MTP sweeps already characterized, which is not worth one GPU job per point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2430