From 1aaedcf22759ecf4806aceafe3a8a2b3542ad16a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 16:00:01 -0500 Subject: [PATCH 1/2] fix(agentx): use full-response interactivity --- docs/data-pipeline.md | 14 ++ package.json | 1 + .../app/src/lib/benchmark-transform.test.ts | 22 +++ packages/app/src/lib/benchmark-transform.ts | 28 ++- packages/constants/src/metric-keys.ts | 18 ++ packages/db/package.json | 1 + .../backfill-full-response-interactivity.ts | 93 +++++++++ packages/db/src/etl/agentic-v3-flatten.ts | 16 +- packages/db/src/etl/benchmark-mapper.test.ts | 30 +++ packages/db/src/etl/benchmark-mapper.ts | 4 +- .../etl/full-response-interactivity.test.ts | 110 +++++++++++ .../db/src/etl/full-response-interactivity.ts | 184 ++++++++++++++++++ .../db/src/etl/trace-replay-ingest.test.ts | 10 + packages/db/src/etl/trace-replay-ingest.ts | 14 ++ .../src/etl/trace-replay-worker-pool.test.ts | 2 + .../db/src/etl/trace-replay-worker-smoke.ts | 1 + 16 files changed, 539 insertions(+), 9 deletions(-) create mode 100644 packages/db/src/backfill-full-response-interactivity.ts create mode 100644 packages/db/src/etl/full-response-interactivity.test.ts create mode 100644 packages/db/src/etl/full-response-interactivity.ts diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index b12f222b..9d1ce61c 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -95,6 +95,20 @@ AIPerf exports public-dataset provenance in `metadata.dataset`, including the Hu Legacy artifacts without provenance leave any existing mapping untouched. A workflow run can map to only one dataset; conflicting dataset IDs fail ingest rather than silently linking the run to an arbitrary dataset. +### Agentic Full-Response Interactivity + +Agentic charts use full-response inter-token latency as their canonical ITL and +define each interactivity percentile as the reciprocal of the matching ITL +percentile. Current aggregate artifacts provide the namespaced +`*_full_response_itl` fields directly. During ingest those fields replace the +legacy visible-content ITL values in the canonical `*_itl` and `*_intvty` keys. + +For artifacts produced before the aggregate field existed, trace-replay ingest +reconstructs each request's decode interval from the retained lifecycle duration +minus TTFT, then divides by `output_sequence_length - 1`. The same helper powers +the one-time `db:backfill-full-response-interactivity` data migration, keeping +historical rows and newly ingested rows on one definition. + ## Frontend Transform Pipeline ### Why transformBenchmarkRows Exists diff --git a/package.json b/package.json index d4731b6b..e4ef26d0 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "admin:db:migrate": "bun run --cwd packages/db db:migrate", "admin:db:migrate:collectivex": "bun run --cwd packages/db db:migrate:collectivex", "admin:db:apply-overrides": "bun run --cwd packages/db db:apply-overrides", + "admin:db:backfill-full-response-interactivity": "bun run --cwd packages/db db:backfill-full-response-interactivity", "admin:db:reset": "bun run --cwd packages/db db:reset", "admin:db:verify": "bun run --cwd packages/db db:verify" }, diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index e79e05de..95ff1958 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -92,6 +92,28 @@ describe('rowToAggDataEntry', () => { expect(entry.rawMetricKeys).toContain('median_ttft'); }); + it('prefers full-response AgentX ITL and interactivity for artifact overlays', () => { + const entry = rowToAggDataEntry( + makeRow({ + benchmark_type: 'agentic_traces', + metrics: { + median_itl: 0.000003, + median_intvty: 333_333, + median_full_response_itl: 0.005, + median_full_response_intvty: 200, + std_full_response_itl: 0.001, + std_full_response_intvty: 20, + }, + }), + ); + + expect(entry.median_itl).toBe(0.005); + expect(entry.median_tpot).toBe(0.005); + expect(entry.median_intvty).toBe(200); + expect(entry.std_itl).toBe(0.001); + expect(entry.std_intvty).toBe(20); + }); + it('defaults missing metrics to 0', () => { const entry = rowToAggDataEntry(makeRow({ metrics: {} })); expect(entry.tput_per_gpu).toBe(0); diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index 07f26d44..a3b6cbfd 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -34,13 +34,33 @@ import type { BenchmarkRow } from '@/lib/api'; */ function applyAgenticMetricAliases(raw: Record): Record { const m: Record = { ...raw }; + const hasFullResponseItl = ['mean', 'median', 'p75', 'p90', 'p95', 'p99', 'p99.9'].some( + (suffix) => typeof raw[`${suffix}_full_response_itl`] === 'number', + ); for (const suffix of ['mean', 'median', 'p75', 'p90', 'p95', 'p99', 'p99.9']) { - const itl = raw[`${suffix}_itl`]; + const fullResponseItl = raw[`${suffix}_full_response_itl`]; + const itl = + typeof fullResponseItl === 'number' && fullResponseItl > 0 + ? fullResponseItl + : hasFullResponseItl + ? undefined + : raw[`${suffix}_itl`]; const ttlt = raw[`${suffix}_ttlt`]; if (m[`${suffix}_e2el`] === undefined && ttlt !== undefined) m[`${suffix}_e2el`] = ttlt; - if (m[`${suffix}_tpot`] === undefined && itl !== undefined) m[`${suffix}_tpot`] = itl; - if (typeof itl === 'number' && itl > 0) m[`${suffix}_intvty`] = 1 / itl; - else delete m[`${suffix}_intvty`]; + if (typeof itl === 'number' && itl > 0) { + m[`${suffix}_itl`] = itl; + if (m[`${suffix}_tpot`] === undefined) m[`${suffix}_tpot`] = itl; + m[`${suffix}_intvty`] = 1 / itl; + } else { + delete m[`${suffix}_itl`]; + delete m[`${suffix}_intvty`]; + } + } + if (typeof raw.std_full_response_itl === 'number') { + m.std_itl = raw.std_full_response_itl; + } + if (typeof raw.std_full_response_intvty === 'number') { + m.std_intvty = raw.std_full_response_intvty; } return m; } diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index ef5756be..94c5b613 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -62,6 +62,24 @@ export const METRIC_KEYS = new Set([ 'p99_intvty', 'p99.9_intvty', 'std_intvty', + // Full-response AgentX timing. These namespaced fields preserve provenance; + // ingest mirrors them onto the canonical *_itl / *_intvty chart fields. + 'median_full_response_itl', + 'mean_full_response_itl', + 'p75_full_response_itl', + 'p90_full_response_itl', + 'p95_full_response_itl', + 'p99_full_response_itl', + 'p99.9_full_response_itl', + 'std_full_response_itl', + 'median_full_response_intvty', + 'mean_full_response_intvty', + 'p75_full_response_intvty', + 'p90_full_response_intvty', + 'p95_full_response_intvty', + 'p99_full_response_intvty', + 'p99.9_full_response_intvty', + 'std_full_response_intvty', // QPS — queries per second (agentic aiperf) 'median_qps', 'mean_qps', diff --git a/packages/db/package.json b/packages/db/package.json index a99339a4..7f676998 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -25,6 +25,7 @@ "db:backfill-aggregate-stats": "bun --env-file=../../.env src/backfill-aggregate-stats.ts", "db:backfill-chart-series": "bun --env-file=../../.env src/backfill-chart-series.ts", "db:backfill-dataset-stats": "bun --env-file=../../.env src/backfill-dataset-stats.ts", + "db:backfill-full-response-interactivity": "bun --env-file=../../.env src/backfill-full-response-interactivity.ts", "db:backfill-request-timeline": "bun --env-file=../../.env src/backfill-request-timeline.ts", "db:backfill-runtime-metadata": "bun --env-file=../../.env src/backfill-runtime-metadata.ts", "db:reset": "bun --env-file=../../.env src/reset-db.ts", diff --git a/packages/db/src/backfill-full-response-interactivity.ts b/packages/db/src/backfill-full-response-interactivity.ts new file mode 100644 index 00000000..74a3cd8f --- /dev/null +++ b/packages/db/src/backfill-full-response-interactivity.ts @@ -0,0 +1,93 @@ +/** + * Backfill canonical AgentX ITL/interactivity from retained AIPerf profiles. + * + * New aggregate artifacts provide `full_response_itl` during normal ingest. + * Historical rows predate that field, but retain the request lifecycle + * timestamps, TTFT, and output token count needed to reconstruct it. + * + * Usage: + * bun run --cwd packages/db db:backfill-full-response-interactivity + * [--limit N] only process the first N candidate benchmark rows + * [--force] recompute rows that already have the namespaced metric + * [--yes] skip the confirmation prompt + */ + +import { hasNoSslFlag } from './cli-utils.js'; +import { createAdminSql, refreshLatestBenchmarks } from './etl/db-utils.js'; +import { fullResponseMetricsFromGzip } from './etl/full-response-interactivity.js'; +import { + confirmProceed, + jsonbParam, + parseLimitForceFlags, + runBackfillMain, + runPerIdBackfill, +} from './lib/backfill-runner.js'; + +const flags = parseLimitForceFlags(); +const sql = createAdminSql({ noSsl: hasNoSslFlag(), max: 1, onnotice: () => {} }); + +async function main(): Promise { + console.log('=== backfill-full-response-interactivity ==='); + console.log(` force = ${flags.force}`); + console.log(` limit = ${flags.limit ?? 'none'}`); + + const candidates = flags.force + ? await sql<{ id: number }[]>` + select br.id + from benchmark_results br + join agentic_trace_replay atr on atr.id = br.trace_replay_id + where br.benchmark_type = 'agentic_traces' + and atr.profile_export_jsonl_gz is not null + order by br.id + ${flags.limit ? sql`limit ${flags.limit}` : sql``} + ` + : await sql<{ id: number }[]>` + select br.id + from benchmark_results br + join agentic_trace_replay atr on atr.id = br.trace_replay_id + where br.benchmark_type = 'agentic_traces' + and atr.profile_export_jsonl_gz is not null + and not (br.metrics ? 'median_full_response_itl') + order by br.id + ${flags.limit ? sql`limit ${flags.limit}` : sql``} + `; + + if (candidates.length === 0) { + console.log('\n Nothing to do — all rows up to date.'); + return; + } + if (!(await confirmProceed(`${candidates.length} candidate benchmark row(s).`))) return; + + await runPerIdBackfill( + candidates.map((candidate) => candidate.id), + async (id) => { + const [row] = await sql<{ profile_export_jsonl_gz: Buffer | null }[]>` + select atr.profile_export_jsonl_gz + from benchmark_results br + join agentic_trace_replay atr on atr.id = br.trace_replay_id + where br.id = ${id} + `; + if (!row) { + console.warn(` id=${id}: row vanished, skipping`); + return 'skipped'; + } + + const patch = fullResponseMetricsFromGzip(row.profile_export_jsonl_gz); + if (Object.keys(patch).length === 0) { + console.warn(` id=${id}: profile has no usable request samples, skipping`); + return 'skipped'; + } + + await sql` + update benchmark_results + set metrics = metrics || ${jsonbParam(sql, patch)} + where id = ${id} + `; + return 'ok'; + }, + ); + + if (process.exitCode !== 1) await refreshLatestBenchmarks(sql); +} + +runBackfillMain('backfill-full-response-interactivity', sql, main); diff --git a/packages/db/src/etl/agentic-v3-flatten.ts b/packages/db/src/etl/agentic-v3-flatten.ts index a3c223af..1349e226 100644 --- a/packages/db/src/etl/agentic-v3-flatten.ts +++ b/packages/db/src/etl/agentic-v3-flatten.ts @@ -27,8 +27,16 @@ const V3_STAT_KEYS: Record = { std: 'std', }; -/** v3 `request_metrics.latency` sub-blocks → flat metric suffix (same name). */ -const V3_LATENCY_METRICS = ['ttft', 'e2el', 'itl', 'tpot', 'intvty'] as const; +/** v3 `request_metrics.latency` sub-blocks → flat metric suffix. */ +const V3_LATENCY_METRICS: Record = { + ttft: 'ttft', + e2el: 'e2el', + itl: 'itl', + tpot: 'tpot', + intvty: 'intvty', + full_response_itl: 'full_response_itl', + full_response_intvty: 'full_response_intvty', +}; /** v3 `request_metrics.tokens` sub-blocks → flat metric suffix. */ const V3_TOKEN_METRICS: Record = { @@ -112,8 +120,8 @@ export function flattenAgenticAggRow(row: Record): Record = {}; // latency distributions - for (const metric of V3_LATENCY_METRICS) { - flattenStatBlock(atPath(row, ['request_metrics', 'latency', metric]), metric, flat); + for (const [metric, suffix] of Object.entries(V3_LATENCY_METRICS)) { + flattenStatBlock(atPath(row, ['request_metrics', 'latency', metric]), suffix, flat); } // qps distribution (window_seconds / samples are intentionally not stats) flattenStatBlock(atPath(row, ['request_metrics', 'qps']), 'qps', flat); diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 9b363719..1985e69d 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -908,6 +908,36 @@ describe('mapBenchmarkRow — v3 agentic nested agg schema', () => { expect(m.std_intvty).toBeCloseTo(24.77636, 6); }); + it('makes full-response ITL canonical when the aggregate provides it', () => { + const tracker = createSkipTracker(); + const row = makeV3AgenticRow(); + row.request_metrics.latency.itl.p99 = 0.2; + row.request_metrics.latency.intvty.p99 = 5; + row.request_metrics.latency.full_response_itl = { + mean: 0.005, + p50: 0.004, + p75: 0.006, + std: 0.001, + }; + row.request_metrics.latency.full_response_intvty = { + mean: 200, + p50: 250, + p75: 166.666667, + std: 20, + }; + + const metrics = mapBenchmarkRow(row, tracker)!.metrics; + + expect(metrics.median_full_response_itl).toBe(0.004); + expect(metrics.p75_full_response_itl).toBe(0.006); + expect(metrics.median_itl).toBe(0.004); + expect(metrics.p75_intvty).toBeCloseTo(1 / 0.006, 6); + expect(metrics.std_itl).toBe(0.001); + expect(metrics.std_intvty).toBe(20); + expect(metrics).not.toHaveProperty('p99_itl'); + expect(metrics).not.toHaveProperty('p99_intvty'); + }); + it("maps kv_offloading 'none' to offload off and skips the empty backend", () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow(makeV3AgenticRow(), tracker); diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index a91103f9..ee71fad2 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -8,6 +8,7 @@ import type { ConfigParams } from './config-cache'; import type { SkipTracker } from './skip-tracker'; import { METRIC_KEYS, PRECISION_KEYS } from '@semianalysisai/inferencex-constants'; import { flattenAgenticAggRow } from './agentic-v3-flatten'; +import { preferFullResponseMetrics } from './full-response-interactivity'; import { resolveModelKey, hwToGpuKey, @@ -241,7 +242,8 @@ export function mapBenchmarkRow( // pool, however, is structural proof of disaggregation and preserves older // Dynamo artifacts that incorrectly emitted disagg=false. const disagg = frameworkDisagg || parallelism.decodeNumWorkers > 0; - const metrics = captureNumericMetrics(row); + let metrics = captureNumericMetrics(row); + if (isAgentic) metrics = preferFullResponseMetrics(metrics); if (!disagg) { const usePrefill = parallelism.decodeTp <= 0 || diff --git a/packages/db/src/etl/full-response-interactivity.test.ts b/packages/db/src/etl/full-response-interactivity.test.ts new file mode 100644 index 00000000..f0c29ca9 --- /dev/null +++ b/packages/db/src/etl/full-response-interactivity.test.ts @@ -0,0 +1,110 @@ +import { gzipSync } from 'node:zlib'; + +import { describe, expect, it } from 'vitest'; + +import { + fullResponseItlSample, + fullResponseMetricsFromGzip, + fullResponseMetricsFromProfile, + preferFullResponseMetrics, +} from './full-response-interactivity'; + +function profileRecord(overrides: Record = {}): string { + return JSON.stringify({ + metadata: { + benchmark_phase: 'profiling', + request_start_ns: 1_000_000_000, + request_end_ns: 146_861_451_000, + }, + metrics: { + output_sequence_length: { value: 26_571, unit: 'tokens' }, + time_to_first_token: { value: 529.058811, unit: 'ms' }, + // This is the legacy mixed-domain value that must not be reused. + inter_token_latency: { value: 0.003067398, unit: 'ms' }, + }, + ...overrides, + }); +} + +describe('fullResponseItlSample', () => { + it('reconstructs the full-response ITL instead of reusing legacy visible-content ITL', () => { + const record = JSON.parse(profileRecord()); + const expected = (145.861451 - 0.529058811) / (26_571 - 1); + + const itl = fullResponseItlSample(record); + + expect(itl).toBeCloseTo(expected, 12); + expect(1 / itl!).toBeCloseTo(182.827, 2); + expect(1 / itl!).toBeLessThan(1_000); + }); + + it('prefers an explicit full-response metric and respects its unit', () => { + const record = JSON.parse( + profileRecord({ + metrics: { + output_sequence_length: { value: 10, unit: 'tokens' }, + full_response_inter_token_latency: { value: 5_500, unit: 'us' }, + full_decode_duration: { value: 100, unit: 's' }, + }, + }), + ); + + expect(fullResponseItlSample(record)).toBeCloseTo(0.0055, 12); + }); +}); + +describe('fullResponseMetricsFromProfile', () => { + it('skips warmup, failed, malformed, and one-token records', () => { + const valid = profileRecord({ + metrics: { + output_sequence_length: { value: 3, unit: 'tokens' }, + full_decode_duration: { value: 20, unit: 'ms' }, + }, + }); + const warmup = profileRecord({ + metadata: { benchmark_phase: 'warmup' }, + metrics: { + output_sequence_length: { value: 3, unit: 'tokens' }, + full_decode_duration: { value: 2, unit: 'ms' }, + }, + }); + const failed = profileRecord({ error: 'server error' }); + const oneToken = profileRecord({ + metrics: { + output_sequence_length: { value: 1, unit: 'tokens' }, + full_decode_duration: { value: 2, unit: 'ms' }, + }, + }); + + const metrics = fullResponseMetricsFromProfile( + [valid, warmup, failed, oneToken, '{invalid-json'].join('\n'), + ); + + expect(metrics.median_full_response_itl).toBeCloseTo(0.01, 12); + expect(metrics.median_itl).toBeCloseTo(0.01, 12); + expect(metrics.median_intvty).toBeCloseTo(100, 12); + }); + + it('reads compressed profile artifacts', () => { + const metrics = fullResponseMetricsFromGzip(gzipSync(profileRecord())); + expect(metrics.median_full_response_intvty).toBeCloseTo(182.827, 2); + }); +}); + +describe('preferFullResponseMetrics', () => { + it('replaces canonical values and removes unmatched legacy percentiles', () => { + const metrics = preferFullResponseMetrics({ + median_itl: 0.000003, + median_intvty: 333_333, + p99_itl: 0.2, + p99_intvty: 5, + median_full_response_itl: 0.005, + median_full_response_intvty: 200, + }); + + expect(metrics.median_itl).toBe(0.005); + expect(metrics.median_intvty).toBe(200); + expect(metrics).not.toHaveProperty('p99_itl'); + expect(metrics).not.toHaveProperty('p99_intvty'); + }); +}); diff --git a/packages/db/src/etl/full-response-interactivity.ts b/packages/db/src/etl/full-response-interactivity.ts new file mode 100644 index 00000000..4958ef68 --- /dev/null +++ b/packages/db/src/etl/full-response-interactivity.ts @@ -0,0 +1,184 @@ +/** + * Canonical AgentX interactivity over the complete client-observed response. + * + * New aggregate artifacts expose `full_response_itl` directly. Older runs can + * reconstruct the same per-request value from the retained AIPerf profile: + * + * (request_end - first_content) / (OSL - 1) + * = (request_end - request_start - TTFT) / (OSL - 1) + * + * All returned latency values are seconds/token. Interactivity is the inverse + * of each matching latency statistic, preserving slow-tail percentile meaning. + */ + +import { gunzipSync } from 'node:zlib'; + +import { meanOf, quantile, readNum } from '../queries/agentic-shared.js'; + +export const FULL_RESPONSE_STAT_KEYS = [ + 'mean', + 'median', + 'p75', + 'p90', + 'p95', + 'p99', + 'p99.9', +] as const; + +type FullResponseStat = (typeof FULL_RESPONSE_STAT_KEYS)[number]; + +interface MetricEnvelope { + value?: unknown; + unit?: unknown; +} + +interface ProfileRecord { + metadata?: { + benchmark_phase?: string; + request_start_ns?: number; + request_end_ns?: number; + }; + metrics?: Record; + error?: unknown; +} + +function secondsFromMetric(value: unknown, defaultUnit: 'ms' | 's' = 'ms'): number | undefined { + const numeric = readNum(value); + if (numeric === undefined || !Number.isFinite(numeric)) return undefined; + const rawUnit = + value && typeof value === 'object' && 'unit' in value + ? String((value as MetricEnvelope).unit ?? '').toLowerCase() + : defaultUnit; + if (['ns', 'nanosecond', 'nanoseconds'].includes(rawUnit)) return numeric / 1e9; + if (['us', 'µs', 'microsecond', 'microseconds'].includes(rawUnit)) return numeric / 1e6; + if (['s', 'sec', 'second', 'seconds'].includes(rawUnit)) return numeric; + return numeric / 1e3; +} + +function populationStd(values: readonly number[]): number { + if (values.length < 2) return 0; + const mean = meanOf([...values]); + return Math.sqrt(meanOf(values.map((value) => (value - mean) ** 2))); +} + +function percentileStats(values: readonly number[]): Record { + const sorted = [...values].toSorted((a, b) => a - b); + return { + mean: meanOf(sorted), + median: quantile(sorted, 0.5), + p75: quantile(sorted, 0.75), + p90: quantile(sorted, 0.9), + p95: quantile(sorted, 0.95), + p99: quantile(sorted, 0.99), + 'p99.9': quantile(sorted, 0.999), + }; +} + +/** Return one full-response ITL sample in seconds/token, or undefined. */ +export function fullResponseItlSample(record: ProfileRecord): number | undefined { + if (record.error) return undefined; + if (record.metadata?.benchmark_phase && record.metadata.benchmark_phase !== 'profiling') { + return undefined; + } + + const metrics = record.metrics ?? {}; + const explicit = secondsFromMetric(metrics.full_response_inter_token_latency); + if (explicit !== undefined && explicit > 0) return explicit; + + const osl = readNum(metrics.output_sequence_length); + if (osl === undefined || osl < 2) return undefined; + + const explicitDuration = secondsFromMetric(metrics.full_decode_duration); + let decodeDuration = explicitDuration; + if (decodeDuration === undefined) { + const ttft = secondsFromMetric(metrics.time_to_first_token); + if (ttft === undefined) return undefined; + + const startNs = record.metadata?.request_start_ns; + const endNs = record.metadata?.request_end_ns; + if ( + typeof startNs === 'number' && + Number.isFinite(startNs) && + typeof endNs === 'number' && + Number.isFinite(endNs) + ) { + decodeDuration = (endNs - startNs) / 1e9 - ttft; + } else { + const httpDuration = secondsFromMetric(metrics.http_req_duration); + if (httpDuration !== undefined) decodeDuration = httpDuration - ttft; + } + } + + if (decodeDuration === undefined || !Number.isFinite(decodeDuration) || decodeDuration <= 0) { + return undefined; + } + return decodeDuration / (osl - 1); +} + +/** + * Compute namespaced and canonical ITL/interactivity fields from a profile. + * Empty/malformed profiles return an empty patch and never erase stored data. + */ +export function fullResponseMetricsFromProfile(jsonl: string): Record { + const samples: number[] = []; + for (const line of jsonl.split('\n')) { + if (!line.trim()) continue; + try { + const sample = fullResponseItlSample(JSON.parse(line) as ProfileRecord); + if (sample !== undefined && Number.isFinite(sample) && sample > 0) samples.push(sample); + } catch { + // A malformed record does not invalidate the remaining profile. + } + } + if (samples.length === 0) return {}; + + const stats = percentileStats(samples); + const patch: Record = {}; + for (const stat of FULL_RESPONSE_STAT_KEYS) { + const itl = stats[stat]; + patch[`${stat}_full_response_itl`] = itl; + patch[`${stat}_full_response_intvty`] = 1 / itl; + } + patch.std_full_response_itl = populationStd(samples); + patch.std_full_response_intvty = populationStd(samples.map((value) => 1 / value)); + return preferFullResponseMetrics(patch); +} + +export function fullResponseMetricsFromGzip(blob: Buffer | null): Record { + if (!blob) return {}; + try { + return fullResponseMetricsFromProfile(gunzipSync(blob).toString('utf8')); + } catch { + return {}; + } +} + +/** + * Replace canonical AgentX ITL/interactivity with namespaced full-response + * values when present. Missing full-response percentiles delete their legacy + * canonical counterparts so one row never mixes the two timing domains. + */ +export function preferFullResponseMetrics(metrics: Record): Record { + const out = { ...metrics }; + const hasFullResponse = FULL_RESPONSE_STAT_KEYS.some( + (stat) => typeof out[`${stat}_full_response_itl`] === 'number', + ); + if (!hasFullResponse) return out; + + for (const stat of FULL_RESPONSE_STAT_KEYS) { + delete out[`${stat}_itl`]; + delete out[`${stat}_intvty`]; + const itl = out[`${stat}_full_response_itl`]; + if (typeof itl === 'number' && Number.isFinite(itl) && itl > 0) { + out[`${stat}_itl`] = itl; + out[`${stat}_intvty`] = 1 / itl; + } + } + delete out.std_itl; + delete out.std_intvty; + const stdItl = out.std_full_response_itl; + const stdIntvty = out.std_full_response_intvty; + if (typeof stdItl === 'number' && Number.isFinite(stdItl)) out.std_itl = stdItl; + if (typeof stdIntvty === 'number' && Number.isFinite(stdIntvty)) out.std_intvty = stdIntvty; + return out; +} diff --git a/packages/db/src/etl/trace-replay-ingest.test.ts b/packages/db/src/etl/trace-replay-ingest.test.ts index a77d87aa..da1906ac 100644 --- a/packages/db/src/etl/trace-replay-ingest.test.ts +++ b/packages/db/src/etl/trace-replay-ingest.test.ts @@ -44,6 +44,12 @@ function preparedFixture(): PreparedTraceReplay { compressionMs: 10, computeMs: 20, cacheHitRates: null, + fullResponseMetrics: { + median_full_response_itl: 0.005, + median_full_response_intvty: 200, + median_itl: 0.005, + median_intvty: 200, + }, }; } @@ -149,5 +155,9 @@ describe('persistPreparedTraceReplay', () => { calls.filter((call) => call.text.includes('trace_replay_upload_parts (field, part, data)')), ).toHaveLength(6); expect(linkCall?.values.some((value) => Array.isArray(value) && value.includes(41))).toBe(true); + const metricUpdate = calls.find((call) => + call.text.includes("not (metrics ? 'median_full_response_itl')"), + ); + expect(metricUpdate?.values).toContain(JSON.stringify(preparedFixture().fullResponseMetrics)); }); }); diff --git a/packages/db/src/etl/trace-replay-ingest.ts b/packages/db/src/etl/trace-replay-ingest.ts index bd8c0403..1a953733 100644 --- a/packages/db/src/etl/trace-replay-ingest.ts +++ b/packages/db/src/etl/trace-replay-ingest.ts @@ -15,6 +15,7 @@ import { createGzip, gzipSync } from 'node:zlib'; import type postgres from 'postgres'; import { computeTraceDerivedPayloads } from './compute-trace-derived.js'; +import { fullResponseMetricsFromGzip } from './full-response-interactivity.js'; import type { ServerMetricsContext } from './server-metrics-adapters'; type Sql = ReturnType; @@ -61,6 +62,7 @@ export interface PreparedTraceReplay { compressionMs: number; computeMs: number; cacheHitRates: { gpu: number; cpu: number | null } | null; + fullResponseMetrics: Record; } function formatBytes(bytes: number | null | undefined): string { @@ -198,6 +200,7 @@ export async function prepareTraceReplay( metricsContext, ); const computeMs = Date.now() - computeStart; + const fullResponseMetrics = fullResponseMetricsFromGzip(profile.data); return { profileGz: profile.data, @@ -214,6 +217,7 @@ export async function prepareTraceReplay( compressionMs, computeMs, cacheHitRates: cacheHitRatesFromChartSeries(chartSeries), + fullResponseMetrics, }; } @@ -244,6 +248,7 @@ export async function persistPreparedTraceReplay( chartSeriesJson, requestTimelineJson, cacheHitRates, + fullResponseMetrics, } = prepared; let linkedCount = 0; @@ -367,6 +372,15 @@ export async function persistPreparedTraceReplay( `; log('updated cache-hit metrics from chart series'); } + if (Object.keys(fullResponseMetrics).length > 0) { + await tx` + update benchmark_results + set metrics = metrics || ${JSON.stringify(fullResponseMetrics)}::jsonb + where id = any(${tx.array(unlinked.map((row) => row.id))}::bigint[]) + and not (metrics ? 'median_full_response_itl') + `; + log('filled full-response ITL and interactivity from the AIPerf profile'); + } linkedCount = unlinked.length; }); if (linkedCount > 0) log(`inserted trace_replay payload (${elapsed(insertStart)})`); diff --git a/packages/db/src/etl/trace-replay-worker-pool.test.ts b/packages/db/src/etl/trace-replay-worker-pool.test.ts index e85ef970..9c3c6cef 100644 --- a/packages/db/src/etl/trace-replay-worker-pool.test.ts +++ b/packages/db/src/etl/trace-replay-worker-pool.test.ts @@ -51,6 +51,7 @@ async function traceFixture() { metrics: { input_sequence_length: { value: 128, unit: 'tokens' }, output_sequence_length: { value: 64, unit: 'tokens' }, + time_to_first_token: { value: 0.001, unit: 'ms' }, }, }), ); @@ -104,6 +105,7 @@ function fingerprint(prepared: PreparedTraceReplay) { chartWindows: prepared.chartWindows, timelineRequests: prepared.timelineRequests, cacheHitRates: prepared.cacheHitRates, + fullResponseMetrics: prepared.fullResponseMetrics, }; } diff --git a/packages/db/src/etl/trace-replay-worker-smoke.ts b/packages/db/src/etl/trace-replay-worker-smoke.ts index 33d25efb..8639d9f1 100644 --- a/packages/db/src/etl/trace-replay-worker-smoke.ts +++ b/packages/db/src/etl/trace-replay-worker-smoke.ts @@ -33,6 +33,7 @@ function fingerprint(prepared: PreparedTraceReplay) { chartWindows: prepared.chartWindows, timelineRequests: prepared.timelineRequests, cacheHitRates: prepared.cacheHitRates, + fullResponseMetrics: prepared.fullResponseMetrics, }; } From 7f397f52dbfe663898fc01f59b3651b672299e15 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 16:01:49 -0500 Subject: [PATCH 2/2] fix(build): resolve full-response helper --- packages/db/src/etl/full-response-interactivity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/src/etl/full-response-interactivity.ts b/packages/db/src/etl/full-response-interactivity.ts index 4958ef68..eeb95c7c 100644 --- a/packages/db/src/etl/full-response-interactivity.ts +++ b/packages/db/src/etl/full-response-interactivity.ts @@ -13,7 +13,7 @@ import { gunzipSync } from 'node:zlib'; -import { meanOf, quantile, readNum } from '../queries/agentic-shared.js'; +import { meanOf, quantile, readNum } from '../queries/agentic-shared'; export const FULL_RESPONSE_STAT_KEYS = [ 'mean',