From 70f2d1812a70a69d1c0a4710874c3024d5039dea Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 27 Jul 2026 22:56:41 -0500 Subject: [PATCH 1/4] feat(agentx): add opt-in burst phase starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow saturation recipes to request AIPerf burst phase starts without changing the default faithful replay behavior. Validate the environment value and cover enabled, disabled, and invalid command generation. 中文：允许饱和度测试配置按需启用 AIPerf 阶段起始突发，同时保持默认的忠实回放行为不变。新增环境变量校验，并覆盖启用、禁用和非法值的命令生成测试。 --- benchmarks/benchmark_lib.sh | 14 ++++++++ benchmarks/test_benchmark_lib.py | 60 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 benchmarks/test_benchmark_lib.py diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index aa9cc6304f..d07929f003 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1775,6 +1775,20 @@ build_replay_cmd() { # least one profile turn after warmup. REPLAY_CMD+=" --trajectory-start-min-ratio 0.25" REPLAY_CMD+=" --trajectory-start-max-ratio 0.75" + # Saturation sweeps need the requested lanes to become active inside the + # benchmark window. Agentic replay otherwise preserves the traces' original + # phase-start offsets, which can spread the initial warmup over many hours. + case "${AIPERF_BURST_PHASE_STARTS:-0}" in + 0) + ;; + 1) + REPLAY_CMD+=" --burst-phase-starts" + ;; + *) + echo "ERROR: AIPERF_BURST_PHASE_STARTS must be 0 or 1" >&2 + return 1 + ;; + esac # After the normal t* snapshot warmup, continue those exact trajectories # with one-token outputs and no idle delays for 10 minutes. Profiling begins # only after those requests drain and resumes from the resulting live state. diff --git a/benchmarks/test_benchmark_lib.py b/benchmarks/test_benchmark_lib.py new file mode 100644 index 0000000000..6f2aec1137 --- /dev/null +++ b/benchmarks/test_benchmark_lib.py @@ -0,0 +1,60 @@ +import shlex +import subprocess +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" + + +def build_replay_command(burst_phase_starts: str | None) -> subprocess.CompletedProcess[str]: + burst_setup = ( + "unset AIPERF_BURST_PHASE_STARTS" + if burst_phase_starts is None + else f"export AIPERF_BURST_PHASE_STARTS={shlex.quote(burst_phase_starts)}" + ) + script = f""" +set -euo pipefail +export IS_AGENTIC=1 +export KV_OFFLOADING=none +source {shlex.quote(str(BENCHMARK_LIB))} +export AIPERF_CLI=aiperf +export PORT=8000 +export MODEL=example/model +export CONC=32 +export DURATION=3600 +export AIPERF_FAILED_REQUEST_THRESHOLD=0.10 +export FRAMEWORK=dynamo-vllm +export AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 +export TRACE_SOURCE_FLAG="--public-dataset example" +{burst_setup} +build_replay_cmd /tmp/aiperf-test +printf '%s\\n' "$REPLAY_CMD" +""" + return subprocess.run( + ["bash", "-c", script], + check=False, + capture_output=True, + text=True, + ) + + +def test_build_replay_command_defaults_to_spread_phase_starts() -> None: + result = build_replay_command(None) + + assert result.returncode == 0, result.stderr + assert "--burst-phase-starts" not in result.stdout + + +def test_build_replay_command_can_enable_burst_phase_starts() -> None: + result = build_replay_command("1") + + assert result.returncode == 0, result.stderr + assert "--burst-phase-starts" in result.stdout + + +def test_build_replay_command_rejects_invalid_burst_value() -> None: + result = build_replay_command("yes") + + assert result.returncode != 0 + assert "AIPERF_BURST_PHASE_STARTS must be 0 or 1" in result.stderr From bdace3d37dee5f501799020860881ef10b085072 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 27 Jul 2026 22:57:31 -0500 Subject: [PATCH 2/4] refactor(agentx): default to burst phase starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Start selected AgentX trajectories together by default so configured concurrency is active within the benchmark window. Preserve faithful recorded phase-start spreading as an explicit AIPERF_BURST_PHASE_STARTS=0 override. 中文：默认同步启动选中的 AgentX 轨迹，使配置的并发度能在基准测试窗口内生效；同时保留 AIPERF_BURST_PHASE_STARTS=0，用于显式恢复按记录时间分散启动。 --- benchmarks/benchmark_lib.sh | 8 ++++---- benchmarks/test_benchmark_lib.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index d07929f003..65c622ce38 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1775,10 +1775,10 @@ build_replay_cmd() { # least one profile turn after warmup. REPLAY_CMD+=" --trajectory-start-min-ratio 0.25" REPLAY_CMD+=" --trajectory-start-max-ratio 0.75" - # Saturation sweeps need the requested lanes to become active inside the - # benchmark window. Agentic replay otherwise preserves the traces' original - # phase-start offsets, which can spread the initial warmup over many hours. - case "${AIPERF_BURST_PHASE_STARTS:-0}" in + # Start every selected trajectory together so the requested lanes become + # active inside the benchmark window. Recipes that intentionally measure + # the traces' recorded phase-start ramp can restore it with an explicit 0. + case "${AIPERF_BURST_PHASE_STARTS:-1}" in 0) ;; 1) diff --git a/benchmarks/test_benchmark_lib.py b/benchmarks/test_benchmark_lib.py index 6f2aec1137..6cbc78a61a 100644 --- a/benchmarks/test_benchmark_lib.py +++ b/benchmarks/test_benchmark_lib.py @@ -39,18 +39,18 @@ def build_replay_command(burst_phase_starts: str | None) -> subprocess.Completed ) -def test_build_replay_command_defaults_to_spread_phase_starts() -> None: +def test_build_replay_command_defaults_to_burst_phase_starts() -> None: result = build_replay_command(None) assert result.returncode == 0, result.stderr - assert "--burst-phase-starts" not in result.stdout + assert "--burst-phase-starts" in result.stdout -def test_build_replay_command_can_enable_burst_phase_starts() -> None: - result = build_replay_command("1") +def test_build_replay_command_can_restore_spread_phase_starts() -> None: + result = build_replay_command("0") assert result.returncode == 0, result.stderr - assert "--burst-phase-starts" in result.stdout + assert "--burst-phase-starts" not in result.stdout def test_build_replay_command_rejects_invalid_burst_value() -> None: From eb386d855f5fdfb02cf15e06d74af3efd87d9193 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 27 Jul 2026 23:06:57 -0500 Subject: [PATCH 3/4] refactor(agentx): move phase-start default into AIPerf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the AIPerf submodule to make burst phase starts the default and remove duplicate InferenceX wrapper plumbing and tests. 中文：更新 AIPerf 子模块，将阶段起始突发设为默认行为，并移除 InferenceX 中重复的封装逻辑和测试。 --- benchmarks/benchmark_lib.sh | 14 -------- benchmarks/test_benchmark_lib.py | 60 -------------------------------- utils/aiperf | 2 +- 3 files changed, 1 insertion(+), 75 deletions(-) delete mode 100644 benchmarks/test_benchmark_lib.py diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 65c622ce38..aa9cc6304f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1775,20 +1775,6 @@ build_replay_cmd() { # least one profile turn after warmup. REPLAY_CMD+=" --trajectory-start-min-ratio 0.25" REPLAY_CMD+=" --trajectory-start-max-ratio 0.75" - # Start every selected trajectory together so the requested lanes become - # active inside the benchmark window. Recipes that intentionally measure - # the traces' recorded phase-start ramp can restore it with an explicit 0. - case "${AIPERF_BURST_PHASE_STARTS:-1}" in - 0) - ;; - 1) - REPLAY_CMD+=" --burst-phase-starts" - ;; - *) - echo "ERROR: AIPERF_BURST_PHASE_STARTS must be 0 or 1" >&2 - return 1 - ;; - esac # After the normal t* snapshot warmup, continue those exact trajectories # with one-token outputs and no idle delays for 10 minutes. Profiling begins # only after those requests drain and resumes from the resulting live state. diff --git a/benchmarks/test_benchmark_lib.py b/benchmarks/test_benchmark_lib.py deleted file mode 100644 index 6cbc78a61a..0000000000 --- a/benchmarks/test_benchmark_lib.py +++ /dev/null @@ -1,60 +0,0 @@ -import shlex -import subprocess -from pathlib import Path - - -REPO_ROOT = Path(__file__).resolve().parents[1] -BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" - - -def build_replay_command(burst_phase_starts: str | None) -> subprocess.CompletedProcess[str]: - burst_setup = ( - "unset AIPERF_BURST_PHASE_STARTS" - if burst_phase_starts is None - else f"export AIPERF_BURST_PHASE_STARTS={shlex.quote(burst_phase_starts)}" - ) - script = f""" -set -euo pipefail -export IS_AGENTIC=1 -export KV_OFFLOADING=none -source {shlex.quote(str(BENCHMARK_LIB))} -export AIPERF_CLI=aiperf -export PORT=8000 -export MODEL=example/model -export CONC=32 -export DURATION=3600 -export AIPERF_FAILED_REQUEST_THRESHOLD=0.10 -export FRAMEWORK=dynamo-vllm -export AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 -export TRACE_SOURCE_FLAG="--public-dataset example" -{burst_setup} -build_replay_cmd /tmp/aiperf-test -printf '%s\\n' "$REPLAY_CMD" -""" - return subprocess.run( - ["bash", "-c", script], - check=False, - capture_output=True, - text=True, - ) - - -def test_build_replay_command_defaults_to_burst_phase_starts() -> None: - result = build_replay_command(None) - - assert result.returncode == 0, result.stderr - assert "--burst-phase-starts" in result.stdout - - -def test_build_replay_command_can_restore_spread_phase_starts() -> None: - result = build_replay_command("0") - - assert result.returncode == 0, result.stderr - assert "--burst-phase-starts" not in result.stdout - - -def test_build_replay_command_rejects_invalid_burst_value() -> None: - result = build_replay_command("yes") - - assert result.returncode != 0 - assert "AIPERF_BURST_PHASE_STARTS must be 0 or 1" in result.stderr diff --git a/utils/aiperf b/utils/aiperf index f5ae8134f8..fb39d0df33 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit f5ae8134f863993e412d350e406fbcacfe714a7a +Subproject commit fb39d0df33cdb4ea61f15e2ec41618604c5c8576 From b8ffd768a5990d6f743cd76ab08ebe8ae4ddb6d9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 27 Jul 2026 23:10:23 -0500 Subject: [PATCH 4/4] chore(agentx): pin merged AIPerf AgentX v1.0 commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point the AIPerf submodule at the cquil11/aiperf-agentx-v1.0 merge commit containing the phase-start default change. 中文：将 AIPerf 子模块更新到 cquil11/aiperf-agentx-v1.0 分支中包含阶段起始默认行为变更的合并提交。 --- utils/aiperf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/aiperf b/utils/aiperf index fb39d0df33..208125aca8 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit fb39d0df33cdb4ea61f15e2ec41618604c5c8576 +Subproject commit 208125aca87a438e43e56517e8a3e5096f8c9281