From 914383ed57dbfc89e86c406462209b3b30b44779 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 4 Aug 2026 15:50:53 -0500 Subject: [PATCH] fix(changelog): derive config labels from run data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use benchmark-derived run configs as the source of truth for speculative decoding and disaggregation in changelog labels and highlights. Preserve raw-key parsing only as a fallback for historical runs without config coverage. Add regression coverage for the Kimi K3 H200 MTP case. 中文：以基准测试运行实际产出的配置作为变更记录标签和高亮中投机解码及分离式推理信息的权威来源。仅对缺少配置覆盖数据的历史运行保留原始键解析回退，并补充 Kimi K3 H200 MTP 场景的回归测试。 --- .../src/components/GlobalFilterContext.tsx | 6 +- .../app/src/components/inference/types.ts | 2 + .../components/inference/ui/ScatterGraph.tsx | 7 +- .../inference/ui/WorkflowInfoDisplay.tsx | 3 +- .../utils/changelogFormatters.test.ts | 42 +++++++++ .../inference/utils/changelogFormatters.tsx | 90 ++++++++++++++++--- 6 files changed, 131 insertions(+), 19 deletions(-) diff --git a/packages/app/src/components/GlobalFilterContext.tsx b/packages/app/src/components/GlobalFilterContext.tsx index a2eec74f6..68abaac05 100644 --- a/packages/app/src/components/GlobalFilterContext.tsx +++ b/packages/app/src/components/GlobalFilterContext.tsx @@ -36,7 +36,7 @@ import { import { computeAutoSwitchDecision } from '@/lib/unofficial-run-auto-switch'; import { countCurvesByPrecision, resolveEffectivePrecisions } from '@/lib/default-precisions'; import { resolveEffectiveSequence } from '@/lib/default-sequence'; -import type { AvailabilityRow, WorkflowInfoResponse } from '@/lib/api'; +import type { AvailabilityRow, RunConfigRow, WorkflowInfoResponse } from '@/lib/api'; const RUNDATE_RE = /^\d{4}-\d{2}-\d{2}$/u; const RUNID_RE = /^[A-Za-z0-9_-]{1,64}$/u; @@ -55,6 +55,7 @@ interface RunInfo { runDate: string; runUrl: string; conclusion: string | null; + runConfigs: RunConfigRow[]; changelog?: { entries: { config_keys: string[]; @@ -127,6 +128,9 @@ function buildRunInfo(data: WorkflowInfoResponse): Record { runDate: run.created_at, runUrl: run.html_url ? `${run.html_url}/attempts/${run.run_attempt}` : '', conclusion: run.conclusion, + runConfigs: (data.runConfigs ?? []).filter( + (config) => String(config.github_run_id) === String(run.github_run_id), + ), ...(runChangelogs.length > 0 && { changelog: { entries: runChangelogs.map((c) => ({ diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index a729be5b6..77331ecfb 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -2,6 +2,7 @@ import type React from 'react'; import type { HardwareEntry } from '@/lib/constants'; import type { Model, Sequence } from '@/lib/data-mappings'; +import type { RunConfigRow } from '@/lib/api'; /** * Role of a single worker process in a multinode / disaggregated deployment. @@ -689,6 +690,7 @@ export interface RunInfo { runDate: string; runUrl: string; conclusion: string | null; + runConfigs?: RunConfigRow[]; changelog?: ChangelogMetadata; } diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 9f25b9418..dd235f434 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -88,7 +88,7 @@ import { renderKnownIssueAnnotations, } from '@/components/inference/utils/knownIssueAnnotations'; import { matchesQuickFilters } from '@/components/inference/utils/quickFilters'; -import { changelogConfigToHwKey } from '@/components/inference/utils/changelogFormatters'; +import { resolveChangelogHwKeys } from '@/components/inference/utils/changelogFormatters'; import { buildFrontierContinuations, fitContinuationLabelBaseline, @@ -737,8 +737,9 @@ const ScatterGraph = React.memo( const hwKeys = cl.entries.flatMap((entry: any) => (entry.config_keys ?? entry['config-keys'] ?? []) .filter((key: string) => selectedPrecisions.includes(key.split('-')[1])) - .map(changelogConfigToHwKey) - .filter((key: string | null): key is string => key !== null), + .flatMap((key: string) => + resolveChangelogHwKeys(key, availableRuns[selectedRunId]?.runConfigs ?? []), + ), ); return new Set(hwKeys); } diff --git a/packages/app/src/components/inference/ui/WorkflowInfoDisplay.tsx b/packages/app/src/components/inference/ui/WorkflowInfoDisplay.tsx index 8458d5976..889a4247e 100644 --- a/packages/app/src/components/inference/ui/WorkflowInfoDisplay.tsx +++ b/packages/app/src/components/inference/ui/WorkflowInfoDisplay.tsx @@ -118,6 +118,7 @@ export default function WorkflowInfoDisplay({ .filter((entry) => entry.config_keys.length > 0); return filtered.length > 0 ? { entries: filtered } : null; })(); + const selectedRunConfigs = availableRuns?.[selectedRunId]?.runConfigs ?? []; return (
@@ -229,7 +230,7 @@ export default function WorkflowInfoDisplay({
Updated Configs
    {entry.config_keys.map((key: string) => ( -
  • {formatConfigKeys(key)}
    • +
  • {formatConfigKeys(key, selectedRunConfigs)}
    • ))}
diff --git a/packages/app/src/components/inference/utils/changelogFormatters.test.ts b/packages/app/src/components/inference/utils/changelogFormatters.test.ts index b657e97e2..095b534ab 100644 --- a/packages/app/src/components/inference/utils/changelogFormatters.test.ts +++ b/packages/app/src/components/inference/utils/changelogFormatters.test.ts @@ -1,11 +1,27 @@ import { describe, expect, it } from 'vitest'; +import type { RunConfigRow } from '@/lib/api'; + import { changelogConfigToHwKey, configKeyMatchesHwKey, formatConfigKeys, + resolveChangelogHwKeys, } from './changelogFormatters'; +const kimiH200MtpConfig: RunConfigRow = { + github_run_id: 30781313910, + run_started_at: '2026-08-04T14:00:18Z', + html_url: 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/30781313910', + head_sha: '114c1bd140ba75e082100ad11f34e3cf0adf9e3d', + model: 'kimik3', + precision: 'fp4', + hardware: 'h200', + framework: 'vllm', + spec_method: 'mtp', + disagg: false, +}; + describe('formatConfigKeys', () => { it('formats a standard config key', () => { const result = formatConfigKeys('gptoss-fp8-b200-vllm'); @@ -53,6 +69,12 @@ describe('formatConfigKeys', () => { 'MI355X (MoRI SGLang) DeepSeek-V4-Pro FP4', ); }); + + it('derives MTP from the benchmark config when the changelog key omits it', () => { + expect(formatConfigKeys('kimik3-fp4-h200-vllm-agentic', [kimiH200MtpConfig])).toBe( + 'H200 (vLLM, MTP) Kimi-K3 FP4', + ); + }); }); describe('changelogConfigToHwKey', () => { @@ -109,4 +131,24 @@ describe('configKeyMatchesHwKey', () => { it('rejects completely different framework', () => { expect(configKeyMatchesHwKey('dsr1-fp8-h200-sglang', 'h200_trt')).toBe(false); }); + + it('matches the benchmark-derived spec method when the changelog key is incomplete', () => { + expect( + configKeyMatchesHwKey('kimik3-fp4-h200-vllm-agentic', 'h200_vllm_mtp', [kimiH200MtpConfig]), + ).toBe(true); + }); +}); + +describe('resolveChangelogHwKeys', () => { + it('treats run content as authoritative over a stale MTP key suffix', () => { + expect( + resolveChangelogHwKeys('kimik3-fp4-h200-vllm-agentic-mtp', [ + { ...kimiH200MtpConfig, spec_method: 'none' }, + ]), + ).toEqual(['h200_vllm']); + }); + + it('falls back to the changelog key for historical runs without config coverage', () => { + expect(resolveChangelogHwKeys('dsr1-fp8-h200-sglang-mtp')).toEqual(['h200_sglang_mtp']); + }); }); diff --git a/packages/app/src/components/inference/utils/changelogFormatters.tsx b/packages/app/src/components/inference/utils/changelogFormatters.tsx index 476ed2211..b8823679f 100644 --- a/packages/app/src/components/inference/utils/changelogFormatters.tsx +++ b/packages/app/src/components/inference/utils/changelogFormatters.tsx @@ -7,6 +7,8 @@ import { import { type Precision, MODEL_PREFIX_MAPPING, getPrecisionLabel } from '@/lib/data-mappings'; import { getHardwareConfig } from '@/lib/constants'; +import { buildAvailabilityHwKey } from '@/lib/chart-utils'; +import type { RunConfigRow } from '@/lib/api'; import { getDisplayLabel } from '@/lib/utils'; const CHANGELOG_FRAMEWORK_KEYS = [ @@ -14,6 +16,65 @@ const CHANGELOG_FRAMEWORK_KEYS = [ ...Object.keys(FRAMEWORK_ALIASES), ].toSorted((a, b) => b.length - a.length); +interface ChangelogConfigScope { + model: string; + precision: string; + hardware: string; + framework: string; +} + +function changelogConfigScope(configKey: string): ChangelogConfigScope | null { + const parts = configKey.toLowerCase().split('-'); + const model = parts[0]; + const precision = parts[1]; + const hardware = parts[2]; + const remainder = parts.slice(3).join('-'); + if (!model || !precision || !hardware || !remainder) return null; + + const framework = CHANGELOG_FRAMEWORK_KEYS.find( + (candidate) => remainder === candidate || remainder.startsWith(`${candidate}-`), + ); + if (!framework) return null; + + return { + model, + precision, + hardware, + framework: resolveFrameworkAlias(framework), + }; +} + +function matchingRunConfigs(configKey: string, runConfigs: RunConfigRow[]): RunConfigRow[] { + const scope = changelogConfigScope(configKey); + if (!scope) return []; + + return runConfigs.filter( + (config) => + config.model === scope.model && + config.precision === scope.precision && + config.hardware === scope.hardware && + resolveFrameworkAlias(config.framework) === scope.framework, + ); +} + +/** + * Resolve a changelog key to chart hardware keys using configs actually emitted + * by the workflow run. The text key scopes the changed model/config family; the + * benchmark rows remain authoritative for spec decoding and disaggregation. + */ +export function resolveChangelogHwKeys( + configKey: string, + runConfigs: RunConfigRow[] = [], +): string[] { + const resolved = matchingRunConfigs(configKey, runConfigs).map((config) => + buildAvailabilityHwKey(config.hardware, config.framework, config.spec_method, config.disagg), + ); + if (resolved.length > 0) return [...new Set(resolved)]; + + const fallback = changelogConfigToHwKey(configKey); + return fallback ? [fallback] : []; +} + /** * Convert a changelog config key into the canonical hardware key used by chart * points and the legend. Agentic config keys append scenario details such as @@ -21,19 +82,16 @@ const CHANGELOG_FRAMEWORK_KEYS = [ * framework labels and must not become part of the legend identity. */ export function changelogConfigToHwKey(configKey: string): string | null { - const parts = configKey.toLowerCase().split('-'); - const gpu = parts[2]; - const remainder = parts.slice(3).join('-'); - if (!gpu || !remainder) return null; + const scope = changelogConfigScope(configKey); + if (!scope) return null; + const remainder = configKey.toLowerCase().split('-').slice(3).join('-'); const framework = CHANGELOG_FRAMEWORK_KEYS.find( (candidate) => remainder === candidate || remainder.startsWith(`${candidate}-`), - ); - if (!framework) return null; - + )!; const trailingParts = remainder.slice(framework.length).split('-').filter(Boolean); const specSuffix = trailingParts.includes('mtp') ? '_mtp' : ''; - return `${gpu}_${resolveFrameworkAlias(framework)}${specSuffix}`; + return `${scope.hardware}_${scope.framework}${specSuffix}`; } export function formatChangelogDescription(desc: string | string[]) { @@ -59,19 +117,23 @@ export function formatChangelogDescription(desc: string | string[]) { } /** - * Check if a changelog config key matches a hwKey. - * Normalizes both to hyphen-separated form for comparison. + * Check whether a changelog scope includes a chart hardware key, preferring + * benchmark-derived run configs when they are available. */ -export function configKeyMatchesHwKey(configKey: string, hwKey: string): boolean { - return changelogConfigToHwKey(configKey) === hwKey; +export function configKeyMatchesHwKey( + configKey: string, + hwKey: string, + runConfigs: RunConfigRow[] = [], +): boolean { + return resolveChangelogHwKeys(configKey, runConfigs).includes(hwKey); } -export function formatConfigKeys(key: string) { +export function formatConfigKeys(key: string, runConfigs: RunConfigRow[] = []) { const parts = key.split('-'); const model = parts[0]; const precision = parts[1]; const modelLabel = MODEL_PREFIX_MAPPING[model]; - const hwKey = changelogConfigToHwKey(key); + const hwKey = resolveChangelogHwKeys(key, runConfigs)[0]; if (!hwKey) { const gpu = parts[2]?.toUpperCase() ?? '';