From f0ff198e54f4a214e1839e962ff412b5bfcd6c51 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 10:42:11 -0500 Subject: [PATCH 1/4] test(b300): use deterministic AIPerf lane warmup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 AIPerf 子模块固定到 PR #26 的测试分支，并将 AgentX 预热改为每条并发通道 10 个请求。 --- .gitmodules | 2 +- benchmarks/benchmark_lib.sh | 12 ++++++------ utils/aiperf | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index b026356de5..9deef0cfc9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "utils/aiperf"] path = utils/aiperf url = https://github.com/SemiAnalysisAI/aiperf.git - branch = cquil11/aiperf-agentx-v1.0 + branch = cquil11/deterministic-agentic-warmup-count diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index c5e6fb2c86..35c02a8d66 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1744,14 +1744,13 @@ build_replay_cmd() { # utils/aiperf/docs/tutorials/agentx-mvp.md. local result_dir="$1" local duration="$DURATION" - local cache_warmup_duration="${AIPERF_AGENTIC_CACHE_WARMUP_DURATION:-600}" + local warmup_requests_per_lane="${AIPERF_WARMUP_REQUESTS_PER_LANE:-10}" # Fast mode is an e2e-only feedback preset used before canonical one-hour # sweeps. AIPerf already exposes both controls, so no AIPerf patch is # required. if [[ "${AIPERF_EXPERIMENTAL_FAST:-0}" == "1" ]]; then duration=1200 - cache_warmup_duration=300 fi export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES="${AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES:-0}" @@ -1784,10 +1783,11 @@ build_replay_cmd() { # least one profile turn after warmup. REPLAY_CMD+=" --trajectory-start-min-ratio 0.25" REPLAY_CMD+=" --trajectory-start-max-ratio 0.75" - # After the normal t* snapshot warmup, continue those exact trajectories - # with one-token outputs and no idle delays. Profiling begins only after - # those requests drain and resumes from the resulting live state. - REPLAY_CMD+=" --agentic-cache-warmup-duration $cache_warmup_duration" + # After the normal t* snapshot warmup, advance every trajectory lane by + # the same deterministic number of one-token requests with no idle delay. + # Profiling begins after those requests drain and resumes from the + # resulting live state. + REPLAY_CMD+=" --warmup-requests-per-lane $warmup_requests_per_lane" # Give long-context warmup requests up to 30 minutes to drain before # declaring warmup failed. Recipes whose saturation arms carry a larger # in-flight working set may override via AGENTIC_WARMUP_GRACE_PERIOD diff --git a/utils/aiperf b/utils/aiperf index 381758a88f..3d77dd3456 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 381758a88f5fa6335455ef5b0ded9557c1162814 +Subproject commit 3d77dd34566c9d312c21fb7a96fcb39d803a7f07 From 2ccc0398310d07df9d12dc799e765bf076bd8112 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 10:42:47 -0500 Subject: [PATCH 2/4] test(b300): trigger deterministic warmup sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为 Kimi K3 B300 的每通道 10 请求确定性预热实验触发完整扫描。 --- perf-changelog.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ac1de53c7d..106073ac1c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5191,3 +5191,9 @@ - "Agentic experiment Variant K (probe, of the #2374 Variant G): REMOVES the kimi-k3-container-deps.sh in-container patch (setup_script, script file, and launcher copy) to verify whether the mamba_hybrid index_fill_ dtype patch is still required by the current vllm/vllm-openai:kimi-k3 image — expected to fail at the first decode step if it is" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2391 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic + description: + - "Test deterministic AIPerf warmup on Kimi K3 B300 with 10 requests per lane." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2408 From ba397f1b9de1345010a26ab93c6411eca4d98d85 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 11:42:12 -0500 Subject: [PATCH 3/4] test(agentx): rebase deterministic AIPerf on v1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point the experimental B300 sweep at the deterministic warmup implementation rebuilt directly on the AgentX v1.0 branch. 中文：将实验性 B300 扫描指向直接基于 AgentX v1.0 分支重建的确定性预热实现。 --- utils/aiperf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/aiperf b/utils/aiperf index 3d77dd3456..08ec06877e 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 3d77dd34566c9d312c21fb7a96fcb39d803a7f07 +Subproject commit 08ec06877e94961b54a0ca0374c168a6bb7fa1bd From 99f4929376c577e61df88311f122a9fcbc8d6b3b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 12:31:03 -0500 Subject: [PATCH 4/4] test(agentx): pin main-based deterministic AIPerf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin the AIPerf submodule to the deterministic warmup branch based on current fork main. Enable headless realtime metrics at a fixed 30-second cadence for sweep visibility. 中文：将 AIPerf 子模块固定到基于当前 fork main 的确定性预热分支，并启用每 30 秒输出一次的无界面实时指标，便于观察扫描运行状态。 Signed-off-by: Cam Quilici --- .gitmodules | 2 +- benchmarks/benchmark_lib.sh | 4 ++++ utils/aiperf | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9deef0cfc9..1f1fade726 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "utils/aiperf"] path = utils/aiperf url = https://github.com/SemiAnalysisAI/aiperf.git - branch = cquil11/deterministic-agentic-warmup-count + branch = cquil11/deterministic-agentic-warmup-count-main diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 35c02a8d66..413ac4e162 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1764,6 +1764,9 @@ build_replay_cmd() { # aiperf validates that SERVICE_PROFILE_CONFIGURE_TIMEOUT >= # DATASET_CONFIGURATION_TIMEOUT at startup. Bump it in lockstep. export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800 + # Headless realtime metrics are opt-in on current AIPerf main. Enable the + # rolling TTFT/ITL/throughput block and emit it every 30 seconds. + export AIPERF_UI_REALTIME_METRICS_ENABLED=true REPLAY_CMD="$AIPERF_CLI profile --scenario inferencex-agentx-mvp" REPLAY_CMD+=" --url http://localhost:$PORT" REPLAY_CMD+=" --endpoint /v1/chat/completions" @@ -1772,6 +1775,7 @@ build_replay_cmd() { REPLAY_CMD+=" --model $MODEL" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" + REPLAY_CMD+=" --stats-interval 30" REPLAY_CMD+=" --random-seed 42" # Fail runs once more than 10% of requests error. This keeps known # transient low-rate failures from killing long sweeps while still diff --git a/utils/aiperf b/utils/aiperf index 08ec06877e..29749372a4 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 08ec06877e94961b54a0ca0374c168a6bb7fa1bd +Subproject commit 29749372a46386280f9d049f8cbd8762f894b9a3