diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 94f91c12..f5c6165e 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -67,7 +67,7 @@ function expectNoVisibleDatesOrSnapshot() { // Next.js RSC payload scripts retain evidence dates for reproducibility. // Assert only against rendered page text, not serialized script/style data. const clone = body.cloneNode(true) as HTMLElement; - clone.querySelectorAll('script, style').forEach((node) => node.remove()); + clone.querySelectorAll('script, style, .sr-only').forEach((node) => node.remove()); return clone.textContent ?? ''; }) .should((text) => { @@ -114,6 +114,137 @@ function textRect(element: Element) { } describe('Overview page', () => { + it('switches between B200 and 30-day comparison without losing page state', () => { + cy.viewport(1280, 900); + cy.visit('/overview'); + + cy.get('[data-testid="overview-page"]') + .children('[data-testid="overview-comparison-switcher"]') + .should('have.length', 1) + .and('have.class', 'justify-center'); + cy.get('[data-testid="overview-comparison-switcher"]') + .should('have.attr', 'aria-label', 'Compare') + .within(() => { + cy.get('[data-overview-comparison="hardware"]') + .should('have.attr', 'aria-current', 'true') + .and('match', 'span') + .and('have.text', 'vs B200') + .then(([active]) => { + const style = getComputedStyle(active); + expect(style.borderBottomWidth).to.equal('2px'); + expect(style.backgroundColor).to.match(/rgba\(0, 0, 0, 0\)|transparent/); + }); + cy.get('[data-overview-comparison="history"]') + .should('have.attr', 'href', '/overview?compare=30d') + .and('have.text', '30-day change') + .click(); + }); + + cy.location('search').should('eq', '?compare=30d'); + cy.get('[data-testid="overview-comparison-switcher"]') + .find('[data-overview-comparison="history"]') + .should('have.attr', 'aria-current', 'true') + .and('match', 'span'); + cy.get('[data-testid="overview-desktop-matrix"] thead').should('contain.text', 'B200'); + cy.get('[data-testid="overview-desktop-matrix"] thead').should('not.contain.text', 'Reference'); + + cy.get('[data-testid="overview-tier-switcher"]') + .contains('a', '100') + .should('have.attr', 'href', '/overview?tier=100&compare=30d'); + cy.get('[data-testid="overview-engine-scope-switcher"]') + .find('[data-overview-engine-scope="all"]') + .should('have.attr', 'href', '/overview?engine=all&compare=30d'); + cy.get('[data-testid="language-toggle"]') + .should('have.attr', 'href', '/zh/overview?compare=30d') + .click(); + cy.location('pathname').should('eq', '/zh/overview'); + cy.location('search').should('eq', '?compare=30d'); + cy.get('[data-testid="overview-comparison-switcher"]') + .should('have.attr', 'aria-label', '对比方式') + .within(() => { + cy.get('[data-overview-comparison="hardware"]').should('have.text', '对比 B200'); + cy.get('[data-overview-comparison="history"]') + .should('have.attr', 'aria-current', 'true') + .and('have.text', '30 天变化'); + }); + cy.contains('当前成本及其相对 30–60 天前最近一次有效平台结果的变化。').should('exist'); + cy.contains('缺少有效 30 天对比的平台仅显示当前成本。').should('exist'); + }); + + it('compares each platform with its own validated result from 30–60 days earlier', () => { + cy.viewport(1280, 900); + cy.visit('/overview?compare=30d'); + + desktopModel('Qwen-3.5-397B-A17B', SINGLE_TURN).within(() => { + // Unlike the hardware view, B200 is a normal platform in the historical + // view and receives its own change badge and heat-map tint. + platform('b200') + .find('[data-testid="overview-cost-delta"]') + .as('b200HistoryDelta') + .should('have.attr', 'data-history-status', 'comparable') + .and('have.attr', 'data-cost-polarity', 'cheaper') + .and('contain.text', '-17%'); + cy.get('@b200HistoryDelta') + .should(($badge) => { + expect($badge).not.to.have.attr('aria-label'); + }) + .find('.sr-only') + .should('have.text', '17% cheaper than this platform’s Jun 10 result'); + platform('mi355x') + .find('[data-testid="overview-cost-delta"]') + .should('have.attr', 'data-history-status', 'comparable') + .and('have.attr', 'data-cost-polarity', 'cheaper') + .and('contain.text', '-25%'); + platform('b300').find('[data-testid="overview-cost-delta"]').should('not.exist'); + platform('b300').then(([cell]) => { + expect(getComputedStyle(cell.closest('td')!).backgroundColor).to.match( + /rgba\(0, 0, 0, 0\)|transparent/, + ); + }); + platform('gb200').find('[data-testid="overview-cost-delta"]').should('not.exist'); + }); + + desktopModel('DeepSeek-V4-Pro', AGENTX).within(() => { + platform('b200') + .find('[data-testid="overview-cost-delta"]') + .should('have.attr', 'data-history-status', 'comparable'); + platform('mi355x') + .find('[data-testid="overview-cost-delta"]') + .should('have.attr', 'data-history-status', 'comparable'); + }); + + cy.contains( + 'Current cost and change versus the latest validated platform result 30–60 days earlier.', + ).should('exist'); + cy.get('[data-testid="overview-cost-delta"]').should('not.contain.text', '∞'); + cy.contains('Platforms without a valid 30-day comparison show current cost only.').should( + 'exist', + ); + expectNoVisibleDatesOrSnapshot(); + }); + + it('keeps the historical comparison complete and non-scrolling across desktop, tablet and phone', () => { + for (const width of [320, 390, 768, 1024, 1279, 1280, 1440]) { + cy.viewport(width, 900); + cy.visit('/overview?compare=30d'); + + cy.get('[data-testid="overview-comparison-switcher"]') + .should('be.visible') + .find('[data-overview-comparison]') + .each(($option) => { + expect($option[0].getBoundingClientRect().height).to.be.at.least(44); + }); + cy.get('[data-testid="overview-cost-delta"][data-hardware="b200"]').should('exist'); + expectNoHorizontalOverflow(); + if (width < 1280) { + expectNoHorizontalScroller('overview-mobile-list'); + expectNoHorizontalScroller('overview-comparison-switcher'); + } else { + expectNoHorizontalScroller('overview-desktop-matrix'); + } + } + }); + it('defaults to community engine scope and switches with canonical links preserving tier and locale', () => { cy.viewport(1280, 900); cy.visit('/overview'); diff --git a/packages/app/cypress/fixtures/api/overview-history-rows.json b/packages/app/cypress/fixtures/api/overview-history-rows.json new file mode 100644 index 00000000..3671581f --- /dev/null +++ b/packages/app/cypress/fixtures/api/overview-history-rows.json @@ -0,0 +1,206 @@ +{ + "Qwen-3.5-397B-A17B": [ + { + "id": 1001, + "hardware": "b200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { + "median_intvty": 40, + "tput_per_gpu": 6000, + "output_tput_per_gpu": 667 + }, + "date": "2026-06-10", + "run_url": null + }, + { + "id": 1002, + "hardware": "b200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 24, + "offload_mode": "off", + "image": null, + "metrics": { + "median_intvty": 80, + "tput_per_gpu": 4500, + "output_tput_per_gpu": 500 + }, + "date": "2026-06-10", + "run_url": null + }, + { + "id": 1003, + "hardware": "mi355x", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 12, + "offload_mode": "off", + "image": null, + "metrics": { + "median_intvty": 50, + "tput_per_gpu": 5000, + "output_tput_per_gpu": 568 + }, + "date": "2026-06-12", + "run_url": null + }, + { + "id": 1004, + "hardware": "mi355x", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 24, + "offload_mode": "off", + "image": null, + "metrics": { + "median_intvty": 120, + "tput_per_gpu": 3900, + "output_tput_per_gpu": 443 + }, + "date": "2026-06-12", + "run_url": null + } + ], + "DeepSeek-V4-Pro": [ + { + "id": 1005, + "hardware": "b200", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "agentic_traces", + "isl": null, + "osl": null, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { + "p90_itl": 0.02, + "p90_ttlt": 120, + "tput_per_gpu": 6000, + "output_tput_per_gpu": 667 + }, + "date": "2026-06-10", + "run_url": null + }, + { + "id": 1006, + "hardware": "mi355x", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "agentic_traces", + "isl": null, + "osl": null, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { + "p90_itl": 0.02, + "p90_ttlt": 130, + "tput_per_gpu": 5000, + "output_tput_per_gpu": 556 + }, + "date": "2026-06-10", + "run_url": null + } + ] +} diff --git a/packages/app/cypress/fixtures/api/overview-rows.json b/packages/app/cypress/fixtures/api/overview-rows.json index 37eafb78..fb4a7979 100644 --- a/packages/app/cypress/fixtures/api/overview-rows.json +++ b/packages/app/cypress/fixtures/api/overview-rows.json @@ -680,6 +680,64 @@ "metrics": { "median_intvty": 120, "tput_per_gpu": 5280, "output_tput_per_gpu": 600 }, "date": "2026-07-18", "run_url": null + }, + { + "id": 1005, + "hardware": "gb200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 40, "tput_per_gpu": 7000, "output_tput_per_gpu": 778 }, + "date": "2026-06-15", + "run_url": null + }, + { + "id": 1006, + "hardware": "gb200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 24, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 80, "tput_per_gpu": 5250, "output_tput_per_gpu": 583 }, + "date": "2026-06-15", + "run_url": null } ], "GLM-5.2": [ diff --git a/packages/app/src/app/(dashboard)/overview/page.tsx b/packages/app/src/app/(dashboard)/overview/page.tsx index 072858cf..336b0284 100644 --- a/packages/app/src/app/(dashboard)/overview/page.tsx +++ b/packages/app/src/app/(dashboard)/overview/page.tsx @@ -4,7 +4,11 @@ import { SITE_NAME, SITE_URL } from '@semianalysisai/inferencex-constants'; import { OverviewPageContent } from '@/components/overview/overview-page'; import { enAlternates } from '@/lib/i18n'; -import { resolveOverviewEngineScope, resolveOverviewTier } from '@/lib/overview-data'; +import { + resolveOverviewComparisonMode, + resolveOverviewEngineScope, + resolveOverviewTier, +} from '@/lib/overview-data'; import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; @@ -38,6 +42,7 @@ export default async function OverviewPage({ searchParams }: Props) { const data = await getOverviewPageData( resolveOverviewTier(sp.tier), resolveOverviewEngineScope(sp.engine), + resolveOverviewComparisonMode(sp.compare), ); return ; } diff --git a/packages/app/src/app/zh/(dashboard)/overview/page.tsx b/packages/app/src/app/zh/(dashboard)/overview/page.tsx index aa867f57..90736695 100644 --- a/packages/app/src/app/zh/(dashboard)/overview/page.tsx +++ b/packages/app/src/app/zh/(dashboard)/overview/page.tsx @@ -4,7 +4,11 @@ import { SITE_NAME, SITE_URL } from '@semianalysisai/inferencex-constants'; import { OverviewPageContent } from '@/components/overview/overview-page'; import { ZH_OG_LOCALE, zhAlternates } from '@/lib/i18n'; -import { resolveOverviewEngineScope, resolveOverviewTier } from '@/lib/overview-data'; +import { + resolveOverviewComparisonMode, + resolveOverviewEngineScope, + resolveOverviewTier, +} from '@/lib/overview-data'; import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; @@ -39,6 +43,7 @@ export default async function ZhOverviewPage({ searchParams }: Props) { const data = await getOverviewPageData( resolveOverviewTier(sp.tier), resolveOverviewEngineScope(sp.engine), + resolveOverviewComparisonMode(sp.compare), ); return ; } diff --git a/packages/app/src/components/overview/overview-page.tsx b/packages/app/src/components/overview/overview-page.tsx index 3e1202d9..18a4edfb 100644 --- a/packages/app/src/components/overview/overview-page.tsx +++ b/packages/app/src/components/overview/overview-page.tsx @@ -5,6 +5,7 @@ import type { OverviewPageData } from '@/lib/overview-data'; import { DesktopOverviewMatrix, MobileOverviewList, + OverviewComparisonSwitcher, OverviewEngineScopeSwitcher, OverviewMethodology, OverviewTierSwitcher, @@ -26,7 +27,7 @@ export function OverviewPageContent({ data, locale }: OverviewPageProps) { const formatters = overviewFormatters(locale); return ( -
+
{/* Two rows at every width: the title, then the metric it is @@ -77,12 +78,14 @@ export function OverviewPageContent({ data, locale }: OverviewPageProps) { @@ -90,6 +93,14 @@ export function OverviewPageContent({ data, locale }: OverviewPageProps) {
+ + {/* Official-only summary; uploaded runs remain in the linked dashboard. */} {/* Clipped on phones for the rounded corners; visible from xl so the desktop matrix header can stick to the page as it scrolls. */} @@ -99,14 +110,16 @@ export function OverviewPageContent({ data, locale }: OverviewPageProps) { locale={locale} formatters={formatters} strings={strings} + comparisonMode={data.comparisonMode} /> - +
); diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index c4bdba29..a45eef8d 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -1,6 +1,8 @@ import { OVERVIEW_TIERS, + type OverviewComparisonMode, type OverviewEngineScope, + type OverviewHistoricalComparison, type OverviewModelSummary, type OverviewPlatformResult, type OverviewTier, @@ -9,6 +11,7 @@ import { buildOverviewDashboardHref, detailHref, overviewEngineScopeHref, + overviewHref, overviewTierHref, } from '@/lib/overview-links'; @@ -34,8 +37,15 @@ export const OVERVIEW_STRINGS = { all: 'All Platforms', community: 'Open Source Community Engines (vLLM/SGLang)', }, + comparisonNavLabel: 'Compare', + comparisonOptions: { + hardware: 'vs B200', + history: '30-day change', + }, caption: 'Cost per million total tokens from each platform’s best observed serving envelope for the scenario shown with each model.', + historyCaption: + 'Current cost and change versus the latest validated platform result 30–60 days earlier.', modelHeader: 'Model · Scenario', scenarioLabels: { single_turn_8k1k: '8K/1K', @@ -65,6 +75,11 @@ export const OVERVIEW_STRINGS = { `${pct} ${cheaper ? 'cheaper' : 'more expensive'} than B200`, costDeltaEvenAria: 'About the same cost as B200', noBaselineAria: 'No B200 baseline to compare against', + historicalDeltaAria: (pct: string, cheaper: boolean, baselineDate: string) => + `${pct} ${cheaper ? 'cheaper' : 'more expensive'} than this platform’s ${baselineDate} result`, + historicalEvenAria: (baselineDate: string) => + `About the same cost as this platform’s ${baselineDate} result`, + historyCellStateLegend: 'Platforms without a valid 30-day comparison show current cost only.', referenceHeader: 'Reference', }, zh: { @@ -81,7 +96,13 @@ export const OVERVIEW_STRINGS = { all: '所有平台', community: '开源社区引擎(vLLM/SGLang)', }, + comparisonNavLabel: '对比方式', + comparisonOptions: { + hardware: '对比 B200', + history: '30 天变化', + }, caption: '按各模型标注的场景,基于各平台最佳观测服务包络线计算每百万总 token 成本。', + historyCaption: '当前成本及其相对 30–60 天前最近一次有效平台结果的变化。', modelHeader: '模型 · 场景', scenarioLabels: { single_turn_8k1k: '8K/1K', @@ -109,6 +130,10 @@ export const OVERVIEW_STRINGS = { costDeltaAria: (pct: string, cheaper: boolean) => `比 B200 ${cheaper ? '便宜' : '昂贵'} ${pct}`, costDeltaEvenAria: '与 B200 成本基本持平', noBaselineAria: '缺少可比较的 B200 基线', + historicalDeltaAria: (pct: string, cheaper: boolean, baselineDate: string) => + `比该平台 ${baselineDate} 的结果${cheaper ? '便宜' : '昂贵'} ${pct}`, + historicalEvenAria: (baselineDate: string) => `与该平台 ${baselineDate} 的结果成本基本持平`, + historyCellStateLegend: '缺少有效 30 天对比的平台仅显示当前成本。', referenceHeader: '基准', }, } as const; @@ -185,7 +210,7 @@ function CellMissing({ hardware, reason }: { hardware: string; reason: string }) const COST_DELTA_NEUTRAL_BAND = 0.05; /** Magnitudes at or beyond this saturate the shade ramp. */ const COST_DELTA_SATURATION = 0.5; -// `no-baseline` (∞) is neutral gray, never red/green: a missing B200 baseline +// Missing comparison evidence is neutral gray, never red/green: availability // is not a better/worse judgment. const COST_DELTA_CLASS = { cheaper: 'text-emerald-700 dark:text-emerald-400', @@ -196,7 +221,7 @@ const COST_DELTA_CLASS = { const COST_DELTA_HUE = { cheaper: '16 185 129', pricier: '239 68 68', - // Parity and "no B200 baseline" both read as neutral gray. + // Parity and missing comparison evidence both read as neutral gray. even: '148 163 184', 'no-baseline': '148 163 184', } as const; @@ -205,11 +230,76 @@ const COST_DELTA_NEUTRAL_ALPHA = '0.10'; type CostDeltaPolarity = keyof typeof COST_DELTA_CLASS; +interface DisplayedComparison { + status: Exclude; + pct: number | null; + baselineDate: string | null; +} + +function displayedComparison( + platform: OverviewPlatformResult, + comparisonMode: OverviewComparisonMode, +): DisplayedComparison | null { + if (platform.costPerMtok === null) return null; + if (comparisonMode === 'history') { + const comparison = platform.historicalComparison; + return comparison?.status === 'comparable' && comparison.costDeltaPct !== null + ? { + status: comparison.status, + pct: comparison.costDeltaPct, + baselineDate: comparison.baselineDate, + } + : null; + } + if (platform.hardware === 'b200') return null; + return { + status: platform.costVsB200Pct === null ? 'no_baseline' : 'comparable', + pct: platform.costVsB200Pct, + baselineDate: null, + }; +} + function costDeltaPolarity(pct: number): CostDeltaPolarity { if (Math.abs(pct) < COST_DELTA_NEUTRAL_BAND) return 'even'; return pct < 0 ? 'cheaper' : 'pricier'; } +function comparisonPolarity(comparison: DisplayedComparison): CostDeltaPolarity { + return comparison.status !== 'comparable' || comparison.pct === null + ? 'no-baseline' + : costDeltaPolarity(comparison.pct); +} + +function comparisonAria( + comparison: DisplayedComparison, + comparisonMode: OverviewComparisonMode, + polarity: CostDeltaPolarity, + formatters: Formatters, + strings: OverviewStrings, +): string { + if (comparison.status === 'no_baseline' || comparison.pct === null) { + return strings.noBaselineAria; + } + if (comparisonMode === 'hardware') { + return polarity === 'even' + ? strings.costDeltaEvenAria + : strings.costDeltaAria( + formatters.percentAbs.format(Math.abs(comparison.pct)), + polarity === 'cheaper', + ); + } + + const baselineDate = + comparison.baselineDate === null ? '' : formatters.shortDate(comparison.baselineDate); + return polarity === 'even' + ? strings.historicalEvenAria(baselineDate) + : strings.historicalDeltaAria( + formatters.percentAbs.format(Math.abs(comparison.pct)), + polarity === 'cheaper', + baselineDate, + ); +} + /** Continuous shade: only background alpha tracks the magnitude, so every * cell reads on one ramp instead of stepping through discrete bins. */ function costDeltaAlpha(pct: number): string { @@ -224,48 +314,43 @@ function costDeltaAlpha(pct: number): string { */ export function costDeltaCellStyle( platform: OverviewPlatformResult, + comparisonMode: OverviewComparisonMode = 'hardware', ): { backgroundColor: string } | undefined { - if (platform.costPerMtok === null) return undefined; - // B200 is the baseline: its null delta means "nothing to compare against - // itself", not the ∞ state, so it keeps the reference column's own tint. - if (platform.hardware === 'b200') return undefined; - const pct = platform.costVsB200Pct; - const polarity: CostDeltaPolarity = pct === null ? 'no-baseline' : costDeltaPolarity(pct); + const comparison = displayedComparison(platform, comparisonMode); + if (comparison === null) return undefined; + const { pct } = comparison; + const polarity = comparisonPolarity(comparison); const alpha = pct === null || polarity === 'even' ? COST_DELTA_NEUTRAL_ALPHA : costDeltaAlpha(pct); return { backgroundColor: `rgb(${COST_DELTA_HUE[polarity]} / ${alpha})` }; } -/** Relative-to-B200 badge. `pct === null` means the row's B200 baseline is - * unavailable: the badge shows a neutral `∞` instead of a percentage. */ +/** Relative comparison badge. Missing B200 evidence stays neutral and uses + * `∞` instead of manufacturing a percentage. */ function CostDeltaBadge({ - pct, + comparison, + comparisonMode, hardware, formatters, strings, phoneRow, }: { - pct: number | null; + comparison: DisplayedComparison; + comparisonMode: OverviewComparisonMode; hardware: string; formatters: Formatters; strings: OverviewStrings; phoneRow: boolean; }) { - const polarity: CostDeltaPolarity = pct === null ? 'no-baseline' : costDeltaPolarity(pct); - const aria = - pct === null - ? strings.noBaselineAria - : polarity === 'even' - ? strings.costDeltaEvenAria - : strings.costDeltaAria( - formatters.percentAbs.format(Math.abs(pct)), - polarity === 'cheaper', - ); + const { pct, status } = comparison; + const polarity = comparisonPolarity(comparison); + const aria = comparisonAria(comparison, comparisonMode, polarity, formatters, strings); return ( {/* Fixed cost | delta grids keep comparisons scannable on desktop and phones; @@ -390,9 +478,10 @@ function CellValue({ )} - {member.hardware === 'b200' ? null : ( + {comparison === null ? null : ( @@ -459,14 +550,23 @@ interface SurfaceProps { locale: OverviewLocale; formatters: Formatters; strings: OverviewStrings; + comparisonMode: OverviewComparisonMode; } -export function DesktopOverviewMatrix({ models, locale, formatters, strings }: SurfaceProps) { +export function DesktopOverviewMatrix({ + models, + locale, + formatters, + strings, + comparisonMode, +}: SurfaceProps) { const platforms = models[0]?.platforms ?? []; return (
- + {platforms.map((platform) => ( @@ -486,9 +586,9 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S @@ -523,8 +623,8 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S {model.platforms.map((platform) => ( ))} @@ -543,7 +644,13 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S ); } -export function MobileOverviewList({ models, locale, formatters, strings }: SurfaceProps) { +export function MobileOverviewList({ + models, + locale, + formatters, + strings, + comparisonMode, +}: SurfaceProps) { return (
    {models.map((model) => ( @@ -561,7 +668,7 @@ export function MobileOverviewList({ models, locale, formatters, strings }: Surf key={platform.hardware} data-testid="overview-mobile-platform-row" data-hardware={platform.hardware} - style={costDeltaCellStyle(platform)} + style={costDeltaCellStyle(platform, comparisonMode)} className="grid min-w-0 grid-cols-[4.25rem_minmax(0,1fr)] gap-x-3 border-b border-border/30 py-1.5 last:border-b-0" > @@ -604,11 +712,13 @@ export function MobileOverviewList({ models, locale, formatters, strings }: Surf export function OverviewTierSwitcher({ tier, engineScope, + comparisonMode, locale, strings, }: { tier: OverviewTier; engineScope: OverviewEngineScope; + comparisonMode: OverviewComparisonMode; locale: OverviewLocale; strings: OverviewStrings; }) { @@ -633,7 +743,7 @@ export function OverviewTierSwitcher({ ) : ( {option} @@ -650,11 +760,13 @@ export function OverviewTierSwitcher({ export function OverviewEngineScopeSwitcher({ engineScope, tier, + comparisonMode, locale, strings, }: { engineScope: OverviewEngineScope; tier: OverviewTier; + comparisonMode: OverviewComparisonMode; locale: OverviewLocale; strings: OverviewStrings; }) { @@ -683,7 +795,7 @@ export function OverviewEngineScopeSwitcher({ {strings.engineScopeOptions[option]} @@ -695,13 +807,69 @@ export function OverviewEngineScopeSwitcher({ ); } -export function OverviewMethodology({ strings }: { strings: OverviewStrings }) { +export function OverviewComparisonSwitcher({ + comparisonMode, + engineScope, + tier, + locale, + strings, +}: { + comparisonMode: OverviewComparisonMode; + engineScope: OverviewEngineScope; + tier: OverviewTier; + locale: OverviewLocale; + strings: OverviewStrings; +}) { + const options: OverviewComparisonMode[] = ['hardware', 'history']; + const optionClass = + 'relative inline-flex min-h-11 min-w-[130px] items-center justify-center whitespace-nowrap border-b-2 border-transparent px-4 py-2 text-sm font-semibold text-muted-foreground transition-colors duration-200 hover:border-muted-foreground/30 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring sm:min-w-[140px]'; + return ( + + ); +} + +export function OverviewMethodology({ + strings, + comparisonMode, +}: { + strings: OverviewStrings; + comparisonMode: OverviewComparisonMode; +}) { return (
    -

    {strings.cellStateLegend}

    + {comparisonMode === 'history' ?

    {strings.historyCaption}

    : null} +

    + {comparisonMode === 'history' ? strings.historyCellStateLegend : strings.cellStateLegend} +

    {strings.methodologyNote}

    ); diff --git a/packages/app/src/lib/benchmark-data.server.ts b/packages/app/src/lib/benchmark-data.server.ts index ccc85317..e288596c 100644 --- a/packages/app/src/lib/benchmark-data.server.ts +++ b/packages/app/src/lib/benchmark-data.server.ts @@ -19,3 +19,15 @@ export const getCachedBenchmarks = cachedQuery( 'benchmarks', { blobOnly: true }, ); + +/** Historical overview snapshots use the same line-level as-of semantics as + * the dashboard, cached separately from the absolute-latest read. */ +export const getCachedBenchmarksAsOf = cachedQuery( + (dbModelKeys: string[], date: string) => { + if (FIXTURES_MODE) return Promise.resolve(loadFixture('benchmarks')); + + return getLatestBenchmarks(getDb(), dbModelKeys, date); + }, + 'benchmarks-as-of', + { blobOnly: true }, +); diff --git a/packages/app/src/lib/overview-data.server.test.ts b/packages/app/src/lib/overview-data.server.test.ts index 2135e0a6..e74d08b7 100644 --- a/packages/app/src/lib/overview-data.server.test.ts +++ b/packages/app/src/lib/overview-data.server.test.ts @@ -6,7 +6,12 @@ import type { OverviewPageData } from './overview-data'; /** `throughput` is TOTAL tok/s per GPU — the overview's cost basis. The * output metric is a decoy so a regression to output pricing surfaces here. */ -function row(hardware: string, framework: string, throughput: number): BenchmarkRow { +function row( + hardware: string, + framework: string, + throughput: number, + date = '2026-07-20', +): BenchmarkRow { return { id: throughput, hardware, @@ -33,7 +38,7 @@ function row(hardware: string, framework: string, throughput: number): Benchmark offload_mode: 'off', image: null, metrics: { median_intvty: 50, tput_per_gpu: throughput, output_tput_per_gpu: 123 }, - date: '2026-07-20', + date, run_url: null, }; } @@ -85,8 +90,12 @@ describe('getOverviewPageData engine scope forwarding', () => { it('forwards community scope through live benchmark queries', async () => { const getCachedBenchmarks = vi.fn(() => Promise.resolve(rows)); + const getCachedBenchmarksAsOf = vi.fn(); vi.doMock('@semianalysisai/inferencex-db/connection', () => ({ FIXTURES_MODE: false })); - vi.doMock('@/lib/benchmark-data.server', () => ({ getCachedBenchmarks })); + vi.doMock('@/lib/benchmark-data.server', () => ({ + getCachedBenchmarks, + getCachedBenchmarksAsOf, + })); vi.doMock('@/lib/test-fixtures', () => ({ loadFixture: vi.fn() })); const { getOverviewPageData } = await import('./overview-data.server'); @@ -98,5 +107,70 @@ describe('getOverviewPageData engine scope forwarding', () => { baseline: 'llmd-vllm', }); expect(getCachedBenchmarks).toHaveBeenCalled(); + expect(getCachedBenchmarksAsOf).not.toHaveBeenCalled(); + }); + + it('loads an as-of snapshot for history mode and rejects baselines older than 60 days', async () => { + const currentRows = [...rows, row('b300', 'vllm', 1100)]; + const historicalRows = [ + row('mi355x', 'dynamo-vllm', 800, '2026-06-15'), + row('b200', 'llmd-vllm', 700, '2026-06-15'), + row('b300', 'vllm', 900, '2026-05-20'), + ]; + const getCachedBenchmarks = vi.fn(() => Promise.resolve(currentRows)); + const getCachedBenchmarksAsOf = vi.fn((_keys: string[], _date: string) => + Promise.resolve(historicalRows), + ); + vi.doMock('@semianalysisai/inferencex-db/connection', () => ({ FIXTURES_MODE: false })); + vi.doMock('@/lib/benchmark-data.server', () => ({ + getCachedBenchmarks, + getCachedBenchmarksAsOf, + })); + vi.doMock('@/lib/test-fixtures', () => ({ loadFixture: vi.fn() })); + + const { getOverviewPageData } = await import('./overview-data.server'); + const page = await getOverviewPageData(50, 'community', 'history'); + const qwen = page.models.find((model) => model.model === Model.Qwen3_5); + const mi355x = qwen?.platforms.find(({ hardware }) => hardware === 'mi355x'); + const b300 = qwen?.platforms.find(({ hardware }) => hardware === 'b300'); + + expect(page.comparisonMode).toBe('history'); + expect(page.historicalWindow).toEqual({ + snapshotDate: '2026-07-20', + targetDate: '2026-06-20', + earliestDate: '2026-05-21', + }); + expect(getCachedBenchmarksAsOf).toHaveBeenCalled(); + expect(getCachedBenchmarksAsOf.mock.calls.every(([, date]) => date === '2026-06-20')).toBe( + true, + ); + expect(mi355x?.historicalComparison?.status).toBe('comparable'); + expect(mi355x?.historicalComparison?.baselineDate).toBe('2026-06-15'); + expect(b300?.historicalComparison?.status).toBe('no_baseline'); + }); + + it('loads separate current and historical fixtures in history mode', async () => { + const loadFixture = vi.fn((name: string) => + name === 'overview-history-rows' + ? { [Model.Qwen3_5]: [row('mi355x', 'dynamo-vllm', 800, '2026-06-15')] } + : { [Model.Qwen3_5]: rows }, + ); + const getCachedBenchmarks = vi.fn(); + const getCachedBenchmarksAsOf = vi.fn(); + vi.doMock('@semianalysisai/inferencex-db/connection', () => ({ FIXTURES_MODE: true })); + vi.doMock('@/lib/benchmark-data.server', () => ({ + getCachedBenchmarks, + getCachedBenchmarksAsOf, + })); + vi.doMock('@/lib/test-fixtures', () => ({ loadFixture })); + + const { getOverviewPageData } = await import('./overview-data.server'); + const page = await getOverviewPageData(50, 'community', 'history'); + + expect(page.comparisonMode).toBe('history'); + expect(loadFixture).toHaveBeenCalledWith('overview-rows'); + expect(loadFixture).toHaveBeenCalledWith('overview-history-rows'); + expect(getCachedBenchmarks).not.toHaveBeenCalled(); + expect(getCachedBenchmarksAsOf).not.toHaveBeenCalled(); }); }); diff --git a/packages/app/src/lib/overview-data.server.ts b/packages/app/src/lib/overview-data.server.ts index a5cd42d0..ac1e21e1 100644 --- a/packages/app/src/lib/overview-data.server.ts +++ b/packages/app/src/lib/overview-data.server.ts @@ -2,38 +2,73 @@ import { DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; import { FIXTURES_MODE } from '@semianalysisai/inferencex-db/connection'; import type { BenchmarkRow } from '@/lib/api'; -import { getCachedBenchmarks } from '@/lib/benchmark-data.server'; +import { getCachedBenchmarks, getCachedBenchmarksAsOf } from '@/lib/benchmark-data.server'; import { DEFAULT_MODELS } from '@/lib/data-mappings'; import { + assembleOverviewHistoricalPageData, assembleOverviewPageData, + OVERVIEW_DEFAULT_COMPARISON_MODE, OVERVIEW_PRIMARY_TIER, + overviewHistoricalWindow, + overviewSnapshotDate, + type OverviewComparisonMode, type OverviewEngineScope, type OverviewPageData, type OverviewTier, } from '@/lib/overview-data'; import { loadFixture } from '@/lib/test-fixtures'; +async function loadRowsByModel( + loader: (keys: string[]) => Promise, +): Promise> { + const entries = await Promise.all( + [...DEFAULT_MODELS].map(async (model) => { + const keys = DISPLAY_MODEL_TO_DB[model] ?? []; + const rows = keys.length > 0 ? await loader(keys) : []; + return [model, rows] as const; + }), + ); + return Object.fromEntries(entries); +} + export async function getOverviewPageData( tier: OverviewTier = OVERVIEW_PRIMARY_TIER, engineScope: OverviewEngineScope = 'community', + comparisonMode: OverviewComparisonMode = OVERVIEW_DEFAULT_COMPARISON_MODE, ): Promise { - // Synthetic rows go through the same assembler as the live path, so a - // contract drift breaks the fixture tests instead of stranding the page. - if (FIXTURES_MODE) { - return assembleOverviewPageData( - loadFixture>('overview-rows'), - tier, - engineScope, - ); + // Synthetic rows go through the same assemblers as the live path, so a + // contract drift breaks fixture tests instead of stranding the page. + const currentRowsByModel = FIXTURES_MODE + ? loadFixture>('overview-rows') + : await loadRowsByModel(getCachedBenchmarks); + if (comparisonMode === 'hardware') { + return assembleOverviewPageData(currentRowsByModel, tier, engineScope); } - const entries = await Promise.all( - [...DEFAULT_MODELS].map(async (model) => { - const keys = DISPLAY_MODEL_TO_DB[model] ?? []; - const rows = keys.length > 0 ? await getCachedBenchmarks(keys) : []; - return [model, rows] as const; - }), + const snapshotDate = overviewSnapshotDate(currentRowsByModel, engineScope); + if (snapshotDate === null) { + return { + ...assembleOverviewPageData(currentRowsByModel, tier, engineScope), + comparisonMode: 'history', + }; + } + + const window = overviewHistoricalWindow(snapshotDate); + const unboundedBaselineRows = FIXTURES_MODE + ? loadFixture>('overview-history-rows') + : await loadRowsByModel((keys) => getCachedBenchmarksAsOf(keys, window.targetDate)); + const baselineRowsByModel = Object.fromEntries( + Object.entries(unboundedBaselineRows).map(([model, rows]) => [ + model, + rows.filter((row) => row.date >= window.earliestDate && row.date <= window.targetDate), + ]), ); - return assembleOverviewPageData(Object.fromEntries(entries), tier, engineScope); + return assembleOverviewHistoricalPageData( + currentRowsByModel, + baselineRowsByModel, + window, + tier, + engineScope, + ); } diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index 908246f3..afb17bb0 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -7,13 +7,19 @@ import { dedupeRowsToLatestPerConfig } from '@/components/inference/hooks/useCha import type { BenchmarkRow } from './api'; import { DEFAULT_MODELS, Model, Precision } from './data-mappings'; import { + assembleOverviewHistoricalPageData, assembleOverviewPageData, buildOverviewModelSummary, overviewCostPerMtok, + overviewHistoricalWindow, overviewScenarioForModel, + overviewSnapshotDate, + overviewTierEvidenceDate, + resolveOverviewComparisonMode, resolveOverviewEngineScope, resolveOverviewTier, type OverviewModelSummary, + type OverviewPageData, } from './overview-data'; let nextId = 1; @@ -120,7 +126,28 @@ function headlinePairOf(summary: OverviewModelSummary, id: string) { return candidate === undefined || baseline === undefined ? undefined : { candidate, baseline }; } +function platformFor( + page: OverviewPageData, + model: Model, + scenario: OverviewModelSummary['scenario'], + hardware: string, +) { + return page.models + .find((summary) => summary.model === model && summary.scenario === scenario) + ?.platforms.find((platform) => platform.hardware === hardware); +} + describe('overview engine scope and scenario selection', () => { + it.each([ + [undefined, 'hardware'], + ['hardware', 'hardware'], + ['30d', 'history'], + [['30d'], 'history'], + ['unknown', 'hardware'], + ] as const)('resolves comparison mode %j to %s', (raw, expected) => { + expect(resolveOverviewComparisonMode(raw)).toBe(expected); + }); + it('assigns each active model to its configured scenario', () => { expect(overviewScenarioForModel(Model.Kimi_K3)).toBe('agentx'); expect(overviewScenarioForModel(Model.GLM_5_2)).toBe('agentx'); @@ -594,6 +621,208 @@ describe('overview engine scope and scenario selection', () => { }); }); +describe('overview historical window', () => { + it('anchors the target and floor to the latest database evidence date', () => { + expect(overviewHistoricalWindow('2026-08-03')).toEqual({ + snapshotDate: '2026-08-03', + targetDate: '2026-07-04', + earliestDate: '2026-06-04', + }); + }); + + it('returns the latest date across model buckets', () => { + expect( + overviewSnapshotDate({ + a: [row({ date: '2026-07-30' })], + b: [row({ date: '2026-08-03' })], + }), + ).toBe('2026-08-03'); + }); + + it('ignores newer rows that cannot appear in the overview', () => { + expect( + overviewSnapshotDate({ + qwen: [ + row({ date: '2026-07-30' }), + row({ date: '2026-08-03', hardware: 'h200' }), + row({ date: '2026-08-02', isl: 1024, osl: 1024 }), + ], + }), + ).toBe('2026-07-30'); + }); + + it('ignores newer rows from scenarios excluded by the curated model layout', () => { + expect( + overviewSnapshotDate({ + [Model.Kimi_K2_5]: [ + row({ model: 'kimik2.5', date: '2026-07-30' }), + agenticRow(50, 25, 7650, 850, { + model: 'kimik2.5', + date: '2026-08-03', + }), + ], + }), + ).toBe('2026-07-30'); + }); + + it('anchors the snapshot date to rows visible in the active engine scope', () => { + const rows = { + [Model.Qwen3_5]: [ + row({ date: '2026-07-30', framework: 'sglang' }), + row({ date: '2026-08-03', framework: 'atom' }), + ], + }; + + expect(overviewSnapshotDate(rows, 'community')).toBe('2026-07-30'); + expect(overviewSnapshotDate(rows, 'all')).toBe('2026-08-03'); + }); + + it('returns null when every model bucket is empty', () => { + expect(overviewSnapshotDate({ a: [], b: [] })).toBeNull(); + }); + + it('uses the selected tier evidence date instead of the config latest date', () => { + const read = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([12000, 10000, 8000, 6000], { date: '2026-08-01' }), + ]).platforms[0].read; + + expect( + overviewTierEvidenceDate({ + ...read, + evidenceDate: { from: '2026-07-03', to: '2026-07-04' }, + }), + ).toBe('2026-07-04'); + expect(overviewTierEvidenceDate({ ...read, evidenceDate: null })).toBe('2026-08-01'); + }); +}); + +describe('assembleOverviewHistoricalPageData', () => { + const window = { + snapshotDate: '2026-08-03', + targetDate: '2026-07-04', + earliestDate: '2026-06-04', + } as const; + const currentRows = { + [Model.Qwen3_5]: [ + ...frontier([12000, 10000, 8000, 6000], { + hardware: 'mi355x', + framework: 'sglang', + date: '2026-08-01', + precision: Precision.FP4, + }), + ...frontier([9600, 8000, 6400, 4800], { + hardware: 'b300', + framework: 'sglang', + date: '2026-08-02', + precision: Precision.FP4, + }), + ...frontier([8400, 7000, 5600, 4200], { + hardware: 'gb200', + framework: 'sglang', + date: '2026-07-04', + precision: Precision.FP4, + }), + ], + }; + const baselineRows = { + [Model.Qwen3_5]: [ + ...frontier([9600, 8000, 6400, 4800], { + hardware: 'mi355x', + framework: 'vllm', + date: '2026-06-20', + precision: Precision.FP4, + }), + ...frontier([8400, 7000, 5600, 4200], { + hardware: 'gb200', + framework: 'sglang', + date: '2026-06-20', + precision: Precision.FP4, + }), + ], + }; + + it('compares each current platform envelope with the same historical platform', () => { + const page = assembleOverviewHistoricalPageData( + currentRows, + baselineRows, + window, + 50, + 'community', + ); + const platform = platformFor(page, Model.Qwen3_5, 'single_turn_8k1k', 'mi355x'); + + expect(page.comparisonMode).toBe('history'); + expect(page.historicalWindow).toEqual(window); + expect(platform?.historicalComparison?.status).toBe('comparable'); + expect(platform?.historicalComparison?.costDeltaPct).toBeCloseTo(-0.2, 9); + expect(platform?.historicalComparison?.baselineDate).toBe('2026-06-20'); + }); + + it('marks a priced current cell without a baseline as no_baseline', () => { + const page = assembleOverviewHistoricalPageData( + currentRows, + baselineRows, + window, + 50, + 'community', + ); + const comparison = platformFor( + page, + Model.Qwen3_5, + 'single_turn_8k1k', + 'b300', + )?.historicalComparison; + + expect(comparison).toEqual({ + status: 'no_baseline', + baselineCostPerMtok: null, + costDeltaPct: null, + baselineDate: null, + }); + }); + + it('does not report zero improvement when current evidence is not newer than the cutoff', () => { + const page = assembleOverviewHistoricalPageData( + currentRows, + baselineRows, + window, + 50, + 'community', + ); + const comparison = platformFor( + page, + Model.Qwen3_5, + 'single_turn_8k1k', + 'gb200', + )?.historicalComparison; + + expect(comparison?.status).toBe('no_newer_result'); + expect(comparison?.costDeltaPct).toBeNull(); + }); + + it('allows the winning engine to change between snapshots', () => { + const page = assembleOverviewHistoricalPageData( + currentRows, + baselineRows, + window, + 50, + 'community', + ); + const current = platformFor(page, Model.Qwen3_5, 'single_turn_8k1k', 'mi355x'); + const baseline = buildOverviewModelSummary( + Model.Qwen3_5, + baselineRows[Model.Qwen3_5], + 50, + 'community', + 'single_turn_8k1k', + ).platforms.find((platform) => platform.hardware === 'mi355x'); + + expect(current?.read.config?.framework).toBe('sglang'); + expect(baseline?.read.config?.framework).toBe('vllm'); + expect(current?.historicalComparison?.status).toBe('comparable'); + }); +}); + describe('overview platform selection', () => { it('keeps FP4 platform boundaries when neither side has an exact read', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 9e474025..9f6c95d2 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -20,6 +20,8 @@ export const OVERVIEW_TIERS = [30, 50, 75, 100, 150, 200] as const; export type OverviewTier = (typeof OVERVIEW_TIERS)[number]; export const OVERVIEW_PRIMARY_TIER = 50; export type OverviewEngineScope = 'all' | 'community'; +export type OverviewComparisonMode = 'hardware' | 'history'; +export const OVERVIEW_DEFAULT_COMPARISON_MODE: OverviewComparisonMode = 'hardware'; export type OverviewScenario = 'single_turn_8k1k' | 'agentx'; /** Row order within a model: the single-turn workload first, AgentX below it. */ export const OVERVIEW_SCENARIOS = ['single_turn_8k1k', 'agentx'] as const; @@ -31,6 +33,13 @@ export function resolveOverviewEngineScope( return candidate === 'all' ? 'all' : 'community'; } +export function resolveOverviewComparisonMode( + raw: string | readonly string[] | undefined, +): OverviewComparisonMode { + const candidate = Array.isArray(raw) ? raw[0] : raw; + return candidate === '30d' ? 'history' : OVERVIEW_DEFAULT_COMPARISON_MODE; +} + export function resolveOverviewTier(raw: string | string[] | undefined): OverviewTier { const candidate = Number(Array.isArray(raw) ? raw[0] : raw); return OVERVIEW_TIERS.find((tier) => tier === candidate) ?? OVERVIEW_PRIMARY_TIER; @@ -89,6 +98,15 @@ export type OverviewMissingReason = | 'cannot_reach_at_tier' | 'no_exact_at_tier'; +export type OverviewHistoricalStatus = 'comparable' | 'no_baseline' | 'no_newer_result'; + +export interface OverviewHistoricalComparison { + status: OverviewHistoricalStatus; + baselineCostPerMtok: number | null; + costDeltaPct: number | null; + baselineDate: string | null; +} + export const OVERVIEW_HARDWARE = ['b200', 'mi355x', 'b300', 'gb200', 'gb300'] as const; export interface OverviewPlatformResult { @@ -103,6 +121,7 @@ export interface OverviewPlatformResult { /** Cost delta vs this row's B200 cell; negative = cheaper. Null on the B200 * cell itself and whenever either cost is unavailable. */ costVsB200Pct: number | null; + historicalComparison: OverviewHistoricalComparison | null; } export interface OverviewModelSummary { @@ -116,6 +135,60 @@ export interface OverviewPageData { models: OverviewModelSummary[]; tier: OverviewTier; engineScope: OverviewEngineScope; + comparisonMode: OverviewComparisonMode; + historicalWindow: OverviewHistoricalWindow | null; +} + +export interface OverviewHistoricalWindow { + snapshotDate: string; + targetDate: string; + earliestDate: string; +} + +function overviewScenarioOfRow(row: BenchmarkRow): OverviewScenario | null { + if (row.benchmark_type === 'agentic_traces') return 'agentx'; + if ( + row.benchmark_type === 'single_turn' && + row.isl === OVERVIEW_WORKLOAD.isl && + row.osl === OVERVIEW_WORKLOAD.osl + ) { + return 'single_turn_8k1k'; + } + return null; +} + +function subtractUtcDays(isoDate: string, days: number): string { + const date = new Date(`${isoDate}T00:00:00Z`); + date.setUTCDate(date.getUTCDate() - days); + return date.toISOString().slice(0, 10); +} + +export function overviewSnapshotDate( + rowsByModel: Readonly>, + engineScope: OverviewEngineScope = 'community', +): string | null { + const dates = Object.entries(rowsByModel).flatMap(([modelKey, rows]) => { + const scenarios = overviewScenariosForModel(modelKey as Model, rows); + return overviewEngineRows(rows, engineScope) + .filter((row) => { + const scenario = overviewScenarioOfRow(row); + return ( + (OVERVIEW_HARDWARE as readonly string[]).includes(row.hardware) && + scenario !== null && + scenarios.includes(scenario) + ); + }) + .map((row) => row.date); + }); + return dates.length === 0 ? null : (dates.toSorted().at(-1) ?? null); +} + +export function overviewHistoricalWindow(snapshotDate: string): OverviewHistoricalWindow { + return { + snapshotDate, + targetDate: subtractUtcDays(snapshotDate, 30), + earliestDate: subtractUtcDays(snapshotDate, 60), + }; } const OVERVIEW_SLICE_PRIORITY = [ @@ -269,8 +342,12 @@ interface ConfigTierRead extends OverviewTierRead { const isInRangeTierRead = (read: T): read is T & { value: number } => read.value !== null && read.boundary === 'interpolated'; +export function overviewTierEvidenceDate(read: OverviewTierRead): string | null { + return read.evidenceDate?.to ?? read.config?.latestDate ?? null; +} + function readFreshness(read: ConfigTierRead): string { - return read.evidenceDate?.to ?? read.config.latestDate; + return overviewTierEvidenceDate(read) ?? read.config.latestDate; } function compareTierReads(a: ConfigTierRead, b: ConfigTierRead): number { @@ -416,6 +493,7 @@ function buildPlatformResults( platform.hardware === 'b200' || b200Cost === null || platform.costPerMtok === null ? null : platform.costPerMtok / b200Cost - 1, + historicalComparison: null, })); } @@ -571,5 +649,78 @@ export function assembleOverviewPageData( ), tier, engineScope, + comparisonMode: OVERVIEW_DEFAULT_COMPARISON_MODE, + historicalWindow: null, + }; +} + +function overviewPlatformKey( + model: OverviewModelSummary, + platform: OverviewPlatformResult, +): string { + return `${model.model}|${model.scenario}|${platform.hardware}`; +} + +export function assembleOverviewHistoricalPageData( + currentRowsByModel: Record, + baselineRowsByModel: Record, + window: OverviewHistoricalWindow, + tier: OverviewTier = OVERVIEW_PRIMARY_TIER, + engineScope: OverviewEngineScope = 'community', +): OverviewPageData { + const current = assembleOverviewPageData(currentRowsByModel, tier, engineScope); + const baseline = assembleOverviewPageData(baselineRowsByModel, tier, engineScope); + const baselineByKey = new Map( + baseline.models.flatMap((model) => + model.platforms.map((platform) => [overviewPlatformKey(model, platform), platform] as const), + ), + ); + + return { + ...current, + comparisonMode: 'history', + historicalWindow: window, + models: current.models.map((model) => ({ + ...model, + platforms: model.platforms.map((platform) => { + if (platform.costPerMtok === null) return platform; + + const previous = baselineByKey.get(overviewPlatformKey(model, platform)); + const currentDate = overviewTierEvidenceDate(platform.read); + if (currentDate === null || currentDate <= window.targetDate) { + return { + ...platform, + historicalComparison: { + status: 'no_newer_result', + baselineCostPerMtok: previous?.costPerMtok ?? null, + costDeltaPct: null, + baselineDate: previous === undefined ? null : overviewTierEvidenceDate(previous.read), + }, + }; + } + + if (previous === undefined || previous.costPerMtok === null) { + return { + ...platform, + historicalComparison: { + status: 'no_baseline', + baselineCostPerMtok: null, + costDeltaPct: null, + baselineDate: null, + }, + }; + } + + return { + ...platform, + historicalComparison: { + status: 'comparable', + baselineCostPerMtok: previous.costPerMtok, + costDeltaPct: platform.costPerMtok / previous.costPerMtok - 1, + baselineDate: overviewTierEvidenceDate(previous.read), + }, + }; + }), + })), }; } diff --git a/packages/app/src/lib/overview-links.test.ts b/packages/app/src/lib/overview-links.test.ts index 55a8fb75..9cb8c5f6 100644 --- a/packages/app/src/lib/overview-links.test.ts +++ b/packages/app/src/lib/overview-links.test.ts @@ -169,6 +169,17 @@ describe('overviewHref', () => { expect(overviewHref('en', 50, 'community')).toBe('/overview'); expect(overviewHref('en', 30, 'all')).toBe('/overview?tier=30&engine=all'); }); + + it('adds historical comparison last and omits the default comparison mode', () => { + expect(overviewHref('en', 50, 'community', 'hardware')).toBe('/overview'); + expect(overviewHref('en', 50, 'community', 'history')).toBe('/overview?compare=30d'); + expect(overviewHref('en', 100, 'all', 'history')).toBe( + '/overview?tier=100&engine=all&compare=30d', + ); + expect(overviewHref('zh', 100, 'all', 'history')).toBe( + '/zh/overview?tier=100&engine=all&compare=30d', + ); + }); }); describe('overview switch links', () => { @@ -185,6 +196,12 @@ describe('overview switch links', () => { }, ); + it('preserves historical comparison when changing tiers', () => { + expect(overviewTierHref('en', 75, 'all', 'history')).toBe( + '/overview?tier=75&engine=all&compare=30d', + ); + }); + it.each([ ['en', 'all', 50, '/overview?engine=all'], ['en', 'all', 100, '/overview?tier=100&engine=all'], @@ -197,4 +214,10 @@ describe('overview switch links', () => { ).toBe(expected); }, ); + + it('preserves historical comparison when changing engine scope', () => { + expect(overviewEngineScopeHref('en', 'all', 50, 'history')).toBe( + '/overview?engine=all&compare=30d', + ); + }); }); diff --git a/packages/app/src/lib/overview-links.ts b/packages/app/src/lib/overview-links.ts index d7aa8d5a..1463cac5 100644 --- a/packages/app/src/lib/overview-links.ts +++ b/packages/app/src/lib/overview-links.ts @@ -1,6 +1,8 @@ import { runIdFromRunUrl } from './known-issues'; import { + OVERVIEW_DEFAULT_COMPARISON_MODE, OVERVIEW_PRIMARY_TIER, + type OverviewComparisonMode, type OverviewConfigResult, type OverviewEngineScope, type OverviewModelSummary, @@ -94,11 +96,13 @@ export function overviewHref( locale: 'en' | 'zh', tier: OverviewTier = OVERVIEW_PRIMARY_TIER, engineScope: OverviewEngineScope = 'community', + comparisonMode: OverviewComparisonMode = OVERVIEW_DEFAULT_COMPARISON_MODE, ): string { const base = locale === 'zh' ? '/zh/overview' : '/overview'; const query = new URLSearchParams(); if (tier !== OVERVIEW_PRIMARY_TIER) query.set('tier', String(tier)); if (engineScope !== 'community') query.set('engine', engineScope); + if (comparisonMode === 'history') query.set('compare', '30d'); const search = query.toString(); return search === '' ? base : `${base}?${search}`; } @@ -108,8 +112,9 @@ export function overviewTierHref( locale: 'en' | 'zh', tier: OverviewTier, engineScope: OverviewEngineScope = 'community', + comparisonMode: OverviewComparisonMode = OVERVIEW_DEFAULT_COMPARISON_MODE, ): string { - return overviewHref(locale, tier, engineScope); + return overviewHref(locale, tier, engineScope, comparisonMode); } /** Engine-scope switch preserving the active service tier. */ @@ -117,6 +122,7 @@ export function overviewEngineScopeHref( locale: 'en' | 'zh', engineScope: OverviewEngineScope, tier: OverviewTier = OVERVIEW_PRIMARY_TIER, + comparisonMode: OverviewComparisonMode = OVERVIEW_DEFAULT_COMPARISON_MODE, ): string { - return overviewHref(locale, tier, engineScope); + return overviewHref(locale, tier, engineScope, comparisonMode); } diff --git a/packages/db/src/etl/trace-replay-ingest.test.ts b/packages/db/src/etl/trace-replay-ingest.test.ts index a77d87aa..5cef23f0 100644 --- a/packages/db/src/etl/trace-replay-ingest.test.ts +++ b/packages/db/src/etl/trace-replay-ingest.test.ts @@ -119,7 +119,7 @@ describe('gzipTraceReplayInput', () => { } finally { await rm(dir, { recursive: true, force: true }); } - }); + }, 15_000); }); describe('persistPreparedTraceReplay', () => {
{strings.caption} + {comparisonMode === 'history' ? strings.historyCaption : strings.caption} +
- {platform.hardware === 'b200' + {comparisonMode === 'hardware' && platform.hardware === 'b200' ? `${platform.hardwareLabel} · ${strings.referenceHeader}` : platform.hardwareLabel}