fix(agentx): use full-response interactivity

docs/data-pipeline.md

@@ -95,6 +95,20 @@ AIPerf exports public-dataset provenance in `metadata.dataset`, including the Hu
 
 Legacy artifacts without provenance leave any existing mapping untouched. A workflow run can map to only one dataset; conflicting dataset IDs fail ingest rather than silently linking the run to an arbitrary dataset.
 
+### Agentic Full-Response Interactivity
+
+Agentic charts use full-response inter-token latency as their canonical ITL and
+define each interactivity percentile as the reciprocal of the matching ITL
+percentile. Current aggregate artifacts provide the namespaced
+`*_full_response_itl` fields directly. During ingest those fields replace the
+legacy visible-content ITL values in the canonical `*_itl` and `*_intvty` keys.
+
+For artifacts produced before the aggregate field existed, trace-replay ingest
+reconstructs each request's decode interval from the retained lifecycle duration
+minus TTFT, then divides by `output_sequence_length - 1`. The same helper powers
+the one-time `db:backfill-full-response-interactivity` data migration, keeping
+historical rows and newly ingested rows on one definition.
+
 ## Frontend Transform Pipeline
 
 ### Why transformBenchmarkRows Exists

package.json

@@ -41,6 +41,7 @@
     "admin:db:migrate": "bun run --cwd packages/db db:migrate",
     "admin:db:migrate:collectivex": "bun run --cwd packages/db db:migrate:collectivex",
     "admin:db:apply-overrides": "bun run --cwd packages/db db:apply-overrides",
+    "admin:db:backfill-full-response-interactivity": "bun run --cwd packages/db db:backfill-full-response-interactivity",
     "admin:db:reset": "bun run --cwd packages/db db:reset",
     "admin:db:verify": "bun run --cwd packages/db db:verify"
   },

packages/app/src/lib/benchmark-transform.test.ts

@@ -92,6 +92,28 @@ describe('rowToAggDataEntry', () => {
     expect(entry.rawMetricKeys).toContain('median_ttft');
   });
 
+  it('prefers full-response AgentX ITL and interactivity for artifact overlays', () => {
+    const entry = rowToAggDataEntry(
+      makeRow({
+        benchmark_type: 'agentic_traces',
+        metrics: {
+          median_itl: 0.000003,
+          median_intvty: 333_333,
+          median_full_response_itl: 0.005,
+          median_full_response_intvty: 200,
+          std_full_response_itl: 0.001,
+          std_full_response_intvty: 20,
+        },
+      }),
+    );
+
+    expect(entry.median_itl).toBe(0.005);
+    expect(entry.median_tpot).toBe(0.005);
+    expect(entry.median_intvty).toBe(200);
+    expect(entry.std_itl).toBe(0.001);
+    expect(entry.std_intvty).toBe(20);
+  });
+
   it('defaults missing metrics to 0', () => {
     const entry = rowToAggDataEntry(makeRow({ metrics: {} }));
     expect(entry.tput_per_gpu).toBe(0);

packages/app/src/lib/benchmark-transform.ts

@@ -34,13 +34,33 @@ import type { BenchmarkRow } from '@/lib/api';
  */
 function applyAgenticMetricAliases(raw: Record<string, number>): Record<string, number> {
   const m: Record<string, number> = { ...raw };
+  const hasFullResponseItl = ['mean', 'median', 'p75', 'p90', 'p95', 'p99', 'p99.9'].some(
+    (suffix) => typeof raw[`${suffix}_full_response_itl`] === 'number',
+  );
   for (const suffix of ['mean', 'median', 'p75', 'p90', 'p95', 'p99', 'p99.9']) {
-    const itl = raw[`${suffix}_itl`];
+    const fullResponseItl = raw[`${suffix}_full_response_itl`];
+    const itl =
+      typeof fullResponseItl === 'number' && fullResponseItl > 0
+        ? fullResponseItl
+        : hasFullResponseItl
+          ? undefined
+          : raw[`${suffix}_itl`];
     const ttlt = raw[`${suffix}_ttlt`];
     if (m[`${suffix}_e2el`] === undefined && ttlt !== undefined) m[`${suffix}_e2el`] = ttlt;
-    if (m[`${suffix}_tpot`] === undefined && itl !== undefined) m[`${suffix}_tpot`] = itl;
-    if (typeof itl === 'number' && itl > 0) m[`${suffix}_intvty`] = 1 / itl;
-    else delete m[`${suffix}_intvty`];
+    if (typeof itl === 'number' && itl > 0) {
+      m[`${suffix}_itl`] = itl;
+      if (m[`${suffix}_tpot`] === undefined) m[`${suffix}_tpot`] = itl;
+      m[`${suffix}_intvty`] = 1 / itl;
+    } else {
+      delete m[`${suffix}_itl`];
+      delete m[`${suffix}_intvty`];
+    }
+  }
+  if (typeof raw.std_full_response_itl === 'number') {
+    m.std_itl = raw.std_full_response_itl;
+  }
+  if (typeof raw.std_full_response_intvty === 'number') {
+    m.std_intvty = raw.std_full_response_intvty;
   }
   return m;
 }

packages/constants/src/metric-keys.ts

@@ -62,6 +62,24 @@ export const METRIC_KEYS = new Set([
   'p99_intvty',
   'p99.9_intvty',
   'std_intvty',
+  // Full-response AgentX timing. These namespaced fields preserve provenance;
+  // ingest mirrors them onto the canonical *_itl / *_intvty chart fields.
+  'median_full_response_itl',
+  'mean_full_response_itl',
+  'p75_full_response_itl',
+  'p90_full_response_itl',
+  'p95_full_response_itl',
+  'p99_full_response_itl',
+  'p99.9_full_response_itl',
+  'std_full_response_itl',
+  'median_full_response_intvty',
+  'mean_full_response_intvty',
+  'p75_full_response_intvty',
+  'p90_full_response_intvty',
+  'p95_full_response_intvty',
+  'p99_full_response_intvty',
+  'p99.9_full_response_intvty',
+  'std_full_response_intvty',
   // QPS — queries per second (agentic aiperf)
   'median_qps',
   'mean_qps',

packages/db/package.json

@@ -25,6 +25,7 @@
     "db:backfill-aggregate-stats": "bun --env-file=../../.env src/backfill-aggregate-stats.ts",
     "db:backfill-chart-series": "bun --env-file=../../.env src/backfill-chart-series.ts",
     "db:backfill-dataset-stats": "bun --env-file=../../.env src/backfill-dataset-stats.ts",
+    "db:backfill-full-response-interactivity": "bun --env-file=../../.env src/backfill-full-response-interactivity.ts",
     "db:backfill-request-timeline": "bun --env-file=../../.env src/backfill-request-timeline.ts",
     "db:backfill-runtime-metadata": "bun --env-file=../../.env src/backfill-runtime-metadata.ts",
     "db:reset": "bun --env-file=../../.env src/reset-db.ts",

packages/db/src/backfill-full-response-interactivity.ts

@@ -0,0 +1,93 @@
+/**
+ * Backfill canonical AgentX ITL/interactivity from retained AIPerf profiles.
+ *
+ * New aggregate artifacts provide `full_response_itl` during normal ingest.
+ * Historical rows predate that field, but retain the request lifecycle
+ * timestamps, TTFT, and output token count needed to reconstruct it.
+ *
+ * Usage:
+ *   bun run --cwd packages/db db:backfill-full-response-interactivity
+ *     [--limit N]   only process the first N candidate benchmark rows
+ *     [--force]     recompute rows that already have the namespaced metric
+ *     [--yes]       skip the confirmation prompt
+ */
+
+import { hasNoSslFlag } from './cli-utils.js';
+import { createAdminSql, refreshLatestBenchmarks } from './etl/db-utils.js';
+import { fullResponseMetricsFromGzip } from './etl/full-response-interactivity.js';
+import {
+  confirmProceed,
+  jsonbParam,
+  parseLimitForceFlags,
+  runBackfillMain,
+  runPerIdBackfill,
+} from './lib/backfill-runner.js';
+
+const flags = parseLimitForceFlags();
+const sql = createAdminSql({ noSsl: hasNoSslFlag(), max: 1, onnotice: () => {} });
+
+async function main(): Promise<void> {
+  console.log('=== backfill-full-response-interactivity ===');
+  console.log(`  force = ${flags.force}`);
+  console.log(`  limit = ${flags.limit ?? 'none'}`);
+
+  const candidates = flags.force
+    ? await sql<{ id: number }[]>`
+        select br.id
+        from benchmark_results br
+        join agentic_trace_replay atr on atr.id = br.trace_replay_id
+        where br.benchmark_type = 'agentic_traces'
+          and atr.profile_export_jsonl_gz is not null
+        order by br.id
+        ${flags.limit ? sql`limit ${flags.limit}` : sql``}
+      `
+    : await sql<{ id: number }[]>`
+        select br.id
+        from benchmark_results br
+        join agentic_trace_replay atr on atr.id = br.trace_replay_id
+        where br.benchmark_type = 'agentic_traces'
+          and atr.profile_export_jsonl_gz is not null
+          and not (br.metrics ? 'median_full_response_itl')
+        order by br.id
+        ${flags.limit ? sql`limit ${flags.limit}` : sql``}
+      `;
+
+  if (candidates.length === 0) {
+    console.log('\n  Nothing to do — all rows up to date.');
+    return;
+  }
+  if (!(await confirmProceed(`${candidates.length} candidate benchmark row(s).`))) return;
+
+  await runPerIdBackfill(
+    candidates.map((candidate) => candidate.id),
+    async (id) => {
+      const [row] = await sql<{ profile_export_jsonl_gz: Buffer | null }[]>`
+        select atr.profile_export_jsonl_gz
+        from benchmark_results br
+        join agentic_trace_replay atr on atr.id = br.trace_replay_id
+        where br.id = ${id}
+      `;
+      if (!row) {
+        console.warn(`  id=${id}: row vanished, skipping`);
+        return 'skipped';
+      }
+
+      const patch = fullResponseMetricsFromGzip(row.profile_export_jsonl_gz);
+      if (Object.keys(patch).length === 0) {
+        console.warn(`  id=${id}: profile has no usable request samples, skipping`);
+        return 'skipped';
+      }
+
+      await sql`
+        update benchmark_results
+        set metrics = metrics || ${jsonbParam(sql, patch)}
+        where id = ${id}
+      `;
+      return 'ok';
+    },
+  );
+
+  if (process.exitCode !== 1) await refreshLatestBenchmarks(sql);
+}
+
+runBackfillMain('backfill-full-response-interactivity', sql, main);

packages/db/src/etl/agentic-v3-flatten.ts

@@ -27,8 +27,16 @@ const V3_STAT_KEYS: Record<string, string> = {
   std: 'std',
 };
 
-/** v3 `request_metrics.latency` sub-blocks → flat metric suffix (same name). */
-const V3_LATENCY_METRICS = ['ttft', 'e2el', 'itl', 'tpot', 'intvty'] as const;
+/** v3 `request_metrics.latency` sub-blocks → flat metric suffix. */
+const V3_LATENCY_METRICS: Record<string, string> = {
+  ttft: 'ttft',
+  e2el: 'e2el',
+  itl: 'itl',
+  tpot: 'tpot',
+  intvty: 'intvty',
+  full_response_itl: 'full_response_itl',
+  full_response_intvty: 'full_response_intvty',
+};
 
 /** v3 `request_metrics.tokens` sub-blocks → flat metric suffix. */
 const V3_TOKEN_METRICS: Record<string, string> = {
@@ -112,8 +120,8 @@ export function flattenAgenticAggRow(row: Record<string, any>): Record<string, a
   const flat: Record<string, number> = {};
 
   // latency distributions
-  for (const metric of V3_LATENCY_METRICS) {
-    flattenStatBlock(atPath(row, ['request_metrics', 'latency', metric]), metric, flat);
+  for (const [metric, suffix] of Object.entries(V3_LATENCY_METRICS)) {
+    flattenStatBlock(atPath(row, ['request_metrics', 'latency', metric]), suffix, flat);
   }
   // qps distribution (window_seconds / samples are intentionally not stats)
   flattenStatBlock(atPath(row, ['request_metrics', 'qps']), 'qps', flat);

packages/db/src/etl/benchmark-mapper.test.ts

@@ -908,6 +908,36 @@ describe('mapBenchmarkRow — v3 agentic nested agg schema', () => {
     expect(m.std_intvty).toBeCloseTo(24.77636, 6);
   });
 
+  it('makes full-response ITL canonical when the aggregate provides it', () => {
+    const tracker = createSkipTracker();
+    const row = makeV3AgenticRow();
+    row.request_metrics.latency.itl.p99 = 0.2;
+    row.request_metrics.latency.intvty.p99 = 5;
+    row.request_metrics.latency.full_response_itl = {
+      mean: 0.005,
+      p50: 0.004,
+      p75: 0.006,
+      std: 0.001,
+    };
+    row.request_metrics.latency.full_response_intvty = {
+      mean: 200,
+      p50: 250,
+      p75: 166.666667,
+      std: 20,
+    };
+
+    const metrics = mapBenchmarkRow(row, tracker)!.metrics;
+
+    expect(metrics.median_full_response_itl).toBe(0.004);
+    expect(metrics.p75_full_response_itl).toBe(0.006);
+    expect(metrics.median_itl).toBe(0.004);
+    expect(metrics.p75_intvty).toBeCloseTo(1 / 0.006, 6);
+    expect(metrics.std_itl).toBe(0.001);
+    expect(metrics.std_intvty).toBe(20);
+    expect(metrics).not.toHaveProperty('p99_itl');
+    expect(metrics).not.toHaveProperty('p99_intvty');
+  });
+
   it("maps kv_offloading 'none' to offload off and skips the empty backend", () => {
     const tracker = createSkipTracker();
     const result = mapBenchmarkRow(makeV3AgenticRow(), tracker);

packages/db/src/etl/benchmark-mapper.ts

@@ -8,6 +8,7 @@ import type { ConfigParams } from './config-cache';
 import type { SkipTracker } from './skip-tracker';
 import { METRIC_KEYS, PRECISION_KEYS } from '@semianalysisai/inferencex-constants';
 import { flattenAgenticAggRow } from './agentic-v3-flatten';
+import { preferFullResponseMetrics } from './full-response-interactivity';
 import {
   resolveModelKey,
   hwToGpuKey,
@@ -241,7 +242,8 @@ export function mapBenchmarkRow(
   // pool, however, is structural proof of disaggregation and preserves older
   // Dynamo artifacts that incorrectly emitted disagg=false.
   const disagg = frameworkDisagg || parallelism.decodeNumWorkers > 0;
-  const metrics = captureNumericMetrics(row);
+  let metrics = captureNumericMetrics(row);
+  if (isAgentic) metrics = preferFullResponseMetrics(metrics);
   if (!disagg) {
     const usePrefill =
       parallelism.decodeTp <= 0 ||