From 45d8c08332556a0b4e4a0f6799f490bda23bcd6a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 13:29:58 -0500 Subject: [PATCH 1/9] feat(agentx): aggregate full-response output TPS Signed-off-by: Cam Quilici --- utils/agentic/aggregation/request_metrics.py | 13 ++++++ .../test_process_agentic_result.py | 40 +++++++++++++++++++ utils/aiperf | 2 +- 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/utils/agentic/aggregation/request_metrics.py b/utils/agentic/aggregation/request_metrics.py index 26fc49df17..4dfe55cae5 100644 --- a/utils/agentic/aggregation/request_metrics.py +++ b/utils/agentic/aggregation/request_metrics.py @@ -232,6 +232,14 @@ def compute_throughput_stats( ) -> tuple[dict[str, Any], dict[str, Any]]: input_tokens = extract_per_record_ints(records, "input_sequence_length") output_tokens = extract_per_record_ints(records, "output_sequence_length") + full_response_tps_per_user = extract_per_record_floats( + records, + "full_response_output_token_throughput_per_user", + ) + full_response_tps_per_user_stats = stats_for( + "full_response_output_token_throughput_per_user", + full_response_tps_per_user, + ) starts_ns = [ int(record["metadata"]["request_start_ns"]) for record in records @@ -255,11 +263,16 @@ def compute_throughput_stats( "output_tput_tps": total_output / duration, "total_tput_tps": (total_input + total_output) / duration, "duration_seconds": duration, + **full_response_tps_per_user_stats, } nested = { "input": {"tokens_per_second": flat["input_tput_tps"]}, "output": {"tokens_per_second": flat["output_tput_tps"]}, "total": {"tokens_per_second": flat["total_tput_tps"]}, + "full_response_output_token_throughput_per_user": _nest_stats( + "full_response_output_token_throughput_per_user", + full_response_tps_per_user_stats, + ), "duration_seconds": duration, "per_gpu": {}, } diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 21a7307630..98ea12a31a 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -129,6 +129,7 @@ "input", "output", "total", + "full_response_output_token_throughput_per_user", "duration_seconds", "per_gpu", } @@ -510,6 +511,45 @@ def test_processor_throughput_per_gpu(tmp_path: Path): ) +def test_processor_aggregates_full_response_output_tps_per_user(tmp_path: Path): + result_dir = tmp_path / "results" + artifact = result_dir / "aiperf_artifacts" + artifact.mkdir(parents=True) + + values = (182.82228456622666, 200.0, 250.0) + with open(artifact / "profile_export.jsonl", "w") as f: + for idx, full_response_tps in enumerate(values): + record = _make_record( + conv_id=f"trace-{idx}", + turn_index=0, + isl=100, + osl=26_571, + ttft_ms=529.058811, + e2e_ms=610.559573, + itl_ms=0.003067398, + start_ns=(idx + 1) * 1_000_000_000, + end_ns=(idx + 1) * 1_000_000_000 + 145_861_451_008, + ) + record["metrics"][ + "full_response_output_token_throughput_per_user" + ] = {"value": full_response_tps, "unit": "tokens/s/user"} + f.write(json.dumps(record) + "\n") + + with open(artifact / "profile_export_aiperf.json", "w") as f: + json.dump({"request_count": len(values)}, f) + + agg = _run_processor(result_dir, tmp_path / "out") + metric = agg["request_metrics"]["throughput"][ + "full_response_output_token_throughput_per_user" + ] + + assert metric["mean"] == pytest.approx(sum(values) / len(values), rel=1e-5) + assert metric["p50"] == pytest.approx(200.0) + assert metric["p75"] == pytest.approx(225.0) + assert metric["p90"] == pytest.approx(240.0) + assert metric["p95"] == pytest.approx(245.0) + + def test_processor_surfaces_allocated_cpu_dram(tmp_path: Path): result_dir = _write_fixture(tmp_path) diff --git a/utils/aiperf b/utils/aiperf index b7b16cf851..4a59fda066 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit b7b16cf851885567988a643282266bce74e34437 +Subproject commit 4a59fda0663ec50850ce1750dd54701cc45ec7a1 From 392bc9d80e5981c7cd845af570f9d85b737d8fcd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 15:33:13 -0500 Subject: [PATCH 2/9] feat(agentx): aggregate full-response ITL Signed-off-by: Cam Quilici --- utils/agentic/aggregation/request_metrics.py | 45 ++++++++++++------- .../test_process_agentic_result.py | 44 +++++++++++------- utils/aiperf | 2 +- 3 files changed, 56 insertions(+), 35 deletions(-) diff --git a/utils/agentic/aggregation/request_metrics.py b/utils/agentic/aggregation/request_metrics.py index 4dfe55cae5..4ac8f454e7 100644 --- a/utils/agentic/aggregation/request_metrics.py +++ b/utils/agentic/aggregation/request_metrics.py @@ -129,17 +129,23 @@ def _nest_stats(prefix: str, flat: dict[str, Any]) -> dict[str, Any]: } -def _interactivity_stats(itl_stats: dict[str, Any], itls: list[float]) -> dict[str, float]: +def _interactivity_stats( + itl_stats: dict[str, Any], + itls: list[float], + *, + itl_prefix: str = "itl", + intvty_prefix: str = "intvty", +) -> dict[str, float]: """Derive slow-tail interactivity from the matching ITL statistic.""" out: dict[str, float] = {} for key in ("mean", "p50", "p75", "p90", "p95"): - value = itl_stats.get(f"{key}_itl") + value = itl_stats.get(f"{key}_{itl_prefix}") if isinstance(value, int | float) and not isinstance(value, bool) and value > 0: - out[f"{key}_intvty"] = 1.0 / value + out[f"{key}_{intvty_prefix}"] = 1.0 / value per_request = [1.0 / value for value in itls if value > 0] if per_request: - out["std_intvty"] = ( + out[f"std_{intvty_prefix}"] = ( statistics.pstdev(per_request) if len(per_request) > 1 else 0.0 ) return out @@ -149,11 +155,21 @@ def compute_latency_stats(records: list[dict[str, Any]]) -> tuple[dict[str, Any] ttfts = _ms_to_s(extract_per_record_floats(records, "time_to_first_token")) e2els = _ms_to_s(extract_per_record_floats(records, "request_latency")) itls = _ms_to_s(extract_per_record_floats(records, "inter_token_latency")) + full_response_itls = _ms_to_s( + extract_per_record_floats(records, "full_response_inter_token_latency") + ) ttft_stats = stats_for("ttft", ttfts) e2el_stats = stats_for("e2el", e2els) itl_stats = stats_for("itl", itls) tpot_stats = stats_for("tpot", itls) intvty_stats = _interactivity_stats(itl_stats, itls) + full_response_itl_stats = stats_for("full_response_itl", full_response_itls) + full_response_intvty_stats = _interactivity_stats( + full_response_itl_stats, + full_response_itls, + itl_prefix="full_response_itl", + intvty_prefix="full_response_intvty", + ) flat: dict[str, Any] = {} flat.update(ttft_stats) @@ -161,6 +177,8 @@ def compute_latency_stats(records: list[dict[str, Any]]) -> tuple[dict[str, Any] flat.update(itl_stats) flat.update(tpot_stats) flat.update(intvty_stats) + flat.update(full_response_itl_stats) + flat.update(full_response_intvty_stats) nested = { "ttft": _nest_stats("ttft", ttft_stats), @@ -168,6 +186,12 @@ def compute_latency_stats(records: list[dict[str, Any]]) -> tuple[dict[str, Any] "itl": _nest_stats("itl", itl_stats), "tpot": _nest_stats("tpot", tpot_stats), "intvty": _nest_stats("intvty", intvty_stats), + "full_response_itl": _nest_stats( + "full_response_itl", full_response_itl_stats + ), + "full_response_intvty": _nest_stats( + "full_response_intvty", full_response_intvty_stats + ), } return flat, nested @@ -232,14 +256,6 @@ def compute_throughput_stats( ) -> tuple[dict[str, Any], dict[str, Any]]: input_tokens = extract_per_record_ints(records, "input_sequence_length") output_tokens = extract_per_record_ints(records, "output_sequence_length") - full_response_tps_per_user = extract_per_record_floats( - records, - "full_response_output_token_throughput_per_user", - ) - full_response_tps_per_user_stats = stats_for( - "full_response_output_token_throughput_per_user", - full_response_tps_per_user, - ) starts_ns = [ int(record["metadata"]["request_start_ns"]) for record in records @@ -263,16 +279,11 @@ def compute_throughput_stats( "output_tput_tps": total_output / duration, "total_tput_tps": (total_input + total_output) / duration, "duration_seconds": duration, - **full_response_tps_per_user_stats, } nested = { "input": {"tokens_per_second": flat["input_tput_tps"]}, "output": {"tokens_per_second": flat["output_tput_tps"]}, "total": {"tokens_per_second": flat["total_tput_tps"]}, - "full_response_output_token_throughput_per_user": _nest_stats( - "full_response_output_token_throughput_per_user", - full_response_tps_per_user_stats, - ), "duration_seconds": duration, "per_gpu": {}, } diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 98ea12a31a..4649c870c0 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -123,13 +123,20 @@ "raw", } REQUEST_METRICS_KEYS = {"qps", "latency", "tokens", "throughput", "cache"} -REQUEST_LATENCY_KEYS = {"ttft", "e2el", "itl", "tpot", "intvty"} +REQUEST_LATENCY_KEYS = { + "ttft", + "e2el", + "itl", + "tpot", + "intvty", + "full_response_itl", + "full_response_intvty", +} REQUEST_TOKEN_KEYS = {"input", "output_actual", "output_expected"} REQUEST_THROUGHPUT_KEYS = { "input", "output", "total", - "full_response_output_token_throughput_per_user", "duration_seconds", "per_gpu", } @@ -511,14 +518,14 @@ def test_processor_throughput_per_gpu(tmp_path: Path): ) -def test_processor_aggregates_full_response_output_tps_per_user(tmp_path: Path): +def test_processor_aggregates_full_response_itl_and_interactivity(tmp_path: Path): result_dir = tmp_path / "results" artifact = result_dir / "aiperf_artifacts" artifact.mkdir(parents=True) - values = (182.82228456622666, 200.0, 250.0) + full_response_itls_ms = (5.469791, 5.0, 4.0) with open(artifact / "profile_export.jsonl", "w") as f: - for idx, full_response_tps in enumerate(values): + for idx, full_response_itl_ms in enumerate(full_response_itls_ms): record = _make_record( conv_id=f"trace-{idx}", turn_index=0, @@ -530,24 +537,27 @@ def test_processor_aggregates_full_response_output_tps_per_user(tmp_path: Path): start_ns=(idx + 1) * 1_000_000_000, end_ns=(idx + 1) * 1_000_000_000 + 145_861_451_008, ) - record["metrics"][ - "full_response_output_token_throughput_per_user" - ] = {"value": full_response_tps, "unit": "tokens/s/user"} + record["metrics"]["full_response_inter_token_latency"] = { + "value": full_response_itl_ms, + "unit": "ms", + } f.write(json.dumps(record) + "\n") with open(artifact / "profile_export_aiperf.json", "w") as f: - json.dump({"request_count": len(values)}, f) + json.dump({"request_count": len(full_response_itls_ms)}, f) agg = _run_processor(result_dir, tmp_path / "out") - metric = agg["request_metrics"]["throughput"][ - "full_response_output_token_throughput_per_user" - ] + latency = agg["request_metrics"]["latency"] + full_response_itl = latency["full_response_itl"] + full_response_intvty = latency["full_response_intvty"] - assert metric["mean"] == pytest.approx(sum(values) / len(values), rel=1e-5) - assert metric["p50"] == pytest.approx(200.0) - assert metric["p75"] == pytest.approx(225.0) - assert metric["p90"] == pytest.approx(240.0) - assert metric["p95"] == pytest.approx(245.0) + assert full_response_itl["p50"] == pytest.approx(0.005) + assert full_response_itl["p75"] == pytest.approx(0.00523) + assert full_response_intvty["p50"] == pytest.approx( + 1 / full_response_itl["p50"] + ) + assert full_response_intvty["p75"] == pytest.approx(1 / 0.0052348955) + assert full_response_intvty["p75"] < full_response_intvty["p50"] def test_processor_surfaces_allocated_cpu_dram(tmp_path: Path): diff --git a/utils/aiperf b/utils/aiperf index 4a59fda066..8128d08e7c 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 4a59fda0663ec50850ce1750dd54701cc45ec7a1 +Subproject commit 8128d08e7cfa31c36254f3c3d8c167488d51dde0 From 1985ac7ac22ca15f620795681bcdb3c221ea2772 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 15:38:09 -0500 Subject: [PATCH 3/9] test(agentx): add Kimi K3 full-response regression sweep Signed-off-by: Cam Quilici --- perf-changelog.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4dd819f46d..3fd2cfe94a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5460,3 +5460,12 @@ - "Use vllm/vllm-openai:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 with prefix caching enabled, default KV-cache dtype, FlashInfer TRT-LLM attention with FP8 indexer KV, and the EAGLE3 drafter on FLASH_ATTN." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2328 +- config-keys: + - kimik3-fp4-b300-vllm-agentic + scenario-type: + - agentic-coding + description: + - "Add client-observed full-response inter-token latency and matching interactivity aggregates for AgentX request artifacts." + - "Keep server-reported completion-token counts aligned with the full HTTP response lifecycle when Kimi-K3 stops emitting parsed content at an end-of-message marker but continues raw generation." + - "Exercise the Kimi-K3 B300 concurrency-1 GPU-resident and DRAM-offload points as the targeted regression for the previously inflated visible-content interactivity calculation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2504 From 56356b143c41c13eb645369355fe5483a331a596 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 17:25:24 -0500 Subject: [PATCH 4/9] Pin AIPerf agentx-v1.0.2 --- utils/aiperf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/aiperf b/utils/aiperf index 8128d08e7c..c3e0d0abf6 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 8128d08e7cfa31c36254f3c3d8c167488d51dde0 +Subproject commit c3e0d0abf68d5955b015e9517612c1b518509718 From b9d400db0763e34ac1f44af8dd33bc03b303aba3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 6 Aug 2026 10:37:30 -0500 Subject: [PATCH 5/9] Repin AIPerf agentx-v1.0.2 after squash --- utils/aiperf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/aiperf b/utils/aiperf index c3e0d0abf6..743fb82ab7 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit c3e0d0abf68d5955b015e9517612c1b518509718 +Subproject commit 743fb82ab798dc898fce98d0321db61e7564942a From 5635c001949f3de877001892bf1e0f9ba6925804 Mon Sep 17 00:00:00 2001 From: Cameron Quilici Date: Thu, 6 Aug 2026 14:42:26 -0500 Subject: [PATCH 6/9] Update perf-changelog.yaml --- perf-changelog.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2ca8d1d443..59d37a108f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5484,4 +5484,5 @@ - "Add client-observed full-response inter-token latency and matching interactivity aggregates for AgentX request artifacts." - "Keep server-reported completion-token counts aligned with the full HTTP response lifecycle when Kimi-K3 stops emitting parsed content at an end-of-message marker but continues raw generation." - "Exercise the Kimi-K3 B300 concurrency-1 GPU-resident and DRAM-offload points as the targeted regression for the previously inflated visible-content interactivity calculation." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2504 \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2504 + From a95d9f72784341046085b2596d5f577e81cb54a5 Mon Sep 17 00:00:00 2001 From: Cameron Quilici Date: Thu, 6 Aug 2026 17:45:26 -0500 Subject: [PATCH 7/9] Update perf-changelog.yaml --- perf-changelog.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8a8d7295b4..b2b4935bcd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5492,4 +5492,5 @@ - "Add client-observed full-response inter-token latency and matching interactivity aggregates for AgentX request artifacts." - "Keep server-reported completion-token counts aligned with the full HTTP response lifecycle when Kimi-K3 stops emitting parsed content at an end-of-message marker but continues raw generation." - "Exercise the Kimi-K3 B300 concurrency-1 GPU-resident and DRAM-offload points as the targeted regression for the previously inflated visible-content interactivity calculation." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2504 \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2504 + From 8c66029a8647cd649efdf2000aba00058dcef27d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 6 Aug 2026 22:14:30 -0500 Subject: [PATCH 8/9] Fix concurrency trimming for generated experiment names --- utils/process_changelog.py | 7 ++++--- utils/test_process_changelog.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 735284750a..577b109b0c 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -68,8 +68,9 @@ def trim_conc(entries: list[dict]) -> list[dict]: ``int`` (single-node) or ``list`` (multi-node). Other fields may contain nested dictionaries or lists, such as KV-offload backend metadata. - - Single-node entries: group by every other field and keep only the entry - with the lowest ``conc`` per group. + - Single-node entries: group by every configuration field other than + ``conc`` and the generated ``exp-name``, then keep only the entry with + the lowest ``conc`` per group. - Multi-node entries: trim the ``conc`` list in place to ``[min(conc)]``. """ groups: dict[tuple, list[int]] = {} @@ -87,7 +88,7 @@ def trim_conc(entries: list[dict]) -> list[dict]: sorted( (k, _freeze_config_value(v)) for k, v in entry.items() - if k != "conc" + if k not in {"conc", "exp-name"} ) ) groups.setdefault(key, []).append(len(out)) diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index 146fd9e568..9febfa9444 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -25,12 +25,13 @@ def test_trim_conc_supports_nested_backend_metadata(): }, } entries = [ - {**common, "conc": 8}, - {**common, "conc": 2}, + {**common, "conc": 8, "exp-name": "kimi_tp8_conc8_kvdram"}, + {**common, "conc": 2, "exp-name": "kimi_tp8_conc2_kvdram"}, { **common, "kv-offload-backend": {"name": "lmcache"}, "conc": 4, + "exp-name": "kimi_tp8_conc4_lmcache", }, ] From 6cff39e6eb1ea1b4330de1a79d28cf5aa0cdd184 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 00:34:37 -0500 Subject: [PATCH 9/9] Remove PR 2504 performance changelog entry --- perf-changelog.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a17d6182f1..4a2145e445 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5492,16 +5492,6 @@ - "Image: lmsysorg/sglang:v0.5.16-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2492 -- config-keys: - - kimik3-fp4-b300-vllm-agentic - scenario-type: - - agentic-coding - description: - - "Add client-observed full-response inter-token latency and matching interactivity aggregates for AgentX request artifacts." - - "Keep server-reported completion-token counts aligned with the full HTTP response lifecycle when Kimi-K3 stops emitting parsed content at an end-of-message marker but continues raw generation." - - "Exercise the Kimi-K3 B300 concurrency-1 GPU-resident and DRAM-offload points as the targeted regression for the previously inflated visible-content interactivity calculation." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2504 - - config-keys: - dsv4-fp4-b200-dynamo-vllm-mtp description: