diff --git a/scripts/lib/session-metrics.ts b/scripts/lib/session-metrics.ts new file mode 100644 index 00000000..80ed9999 --- /dev/null +++ b/scripts/lib/session-metrics.ts @@ -0,0 +1,107 @@ +/** + * Pure helpers for parsing Claude Agent SDK session results. + * + * Extracted out of `runSession` in token-benchmark.ts (#1906), which + * exceeded codegraph's cognitive/cyclomatic/maxNesting complexity + * thresholds. The usage-metric field-fallback logic (the SDK returns + * snake_case or camelCase field names depending on version) and the + * nested tool_use-block scan were the biggest contributors — both are + * pure functions of the SDK result, so they're unit-testable here without + * mocking the Agent SDK itself. + */ + +/** + * First truthy value among `values` (falsy values, including 0, are + * skipped), else the last value. Equivalent to `a || b || ... || z`. + */ +export function firstTruthy(...values: T[]): T { + for (const v of values) { + if (v) return v; + } + return values[values.length - 1]; +} + +export interface UsageMetrics { + inputTokens: number; + outputTokens: number; + cacheReadInputTokens: number; + totalCostUsd: number; + numTurns: number; +} + +export interface SessionResult { + usage?: Record; + num_turns?: number; + numTurns?: number; +} + +/** + * Extract usage/turn metrics from an Agent SDK query result, tolerating + * both snake_case (raw API) and camelCase (SDK-normalized) field names. + */ +export function extractUsageMetrics(result: SessionResult): UsageMetrics { + const usage = result.usage || {}; + return { + inputTokens: firstTruthy(usage.input_tokens, usage.inputTokens, 0), + outputTokens: firstTruthy(usage.output_tokens, usage.outputTokens, 0), + cacheReadInputTokens: firstTruthy( + usage.cache_read_input_tokens, + usage.cacheReadInputTokens, + 0, + ), + totalCostUsd: + Math.round(firstTruthy(usage.total_cost_usd, usage.totalCostUsd, 0) * 100) / 100, + numTurns: firstTruthy(result.num_turns, result.numTurns, 0), + }; +} + +export interface ToolUseBlock { + type: string; + name?: string; + input?: { file_path?: string }; +} + +export interface AssistantMessage { + role: string; + content?: unknown; +} + +/** + * Collect all `tool_use` content blocks from a session's assistant + * messages, in message order. + */ +export function collectToolUseBlocks(messages: AssistantMessage[]): ToolUseBlock[] { + const blocks: ToolUseBlock[] = []; + for (const msg of messages) { + if (msg.role !== 'assistant') continue; + const msgBlocks = Array.isArray(msg.content) ? (msg.content as ToolUseBlock[]) : []; + for (const block of msgBlocks) { + if (block.type === 'tool_use') blocks.push(block); + } + } + return blocks; +} + +export interface ToolCallTally { + toolCalls: Record; + uniqueFilesRead: number; +} + +/** + * Tally tool-call counts by name and the set of unique files read, from a + * session's `tool_use` blocks. + */ +export function tallyToolCalls(messages: AssistantMessage[]): ToolCallTally { + const toolCalls: Record = {}; + const uniqueFilesRead = new Set(); + + for (const block of collectToolUseBlocks(messages)) { + const name = block.name || 'unknown'; + toolCalls[name] = (toolCalls[name] || 0) + 1; + if (name === 'Read' && block.input?.file_path) { + uniqueFilesRead.add(block.input.file_path); + } + } + + return { toolCalls, uniqueFilesRead: uniqueFilesRead.size }; +} diff --git a/scripts/token-benchmark.ts b/scripts/token-benchmark.ts index 15bf5692..8d2f4c16 100644 --- a/scripts/token-benchmark.ts +++ b/scripts/token-benchmark.ts @@ -28,6 +28,7 @@ import { parseArgs } from 'node:util'; import { ISSUES, extractAgentOutput, validateResult } from './token-benchmark-issues.js'; import { getBenchmarkVersion } from './bench-version.js'; import { median, round1, timeMedian } from './lib/bench-timing.js'; +import { extractUsageMetrics, tallyToolCalls } from './lib/session-metrics.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(__dirname, '..'); @@ -167,21 +168,15 @@ async function buildCodegraph(nextjsDir) { // ── Session runner ──────────────────────────────────────────────────────── /** - * Run a single agent session using the Claude Agent SDK. + * Build the Agent SDK `options` for a single session. + * Codegraph mode additionally wires up the codegraph MCP server against + * the graph already built for `nextjsDir`. * * @param {'baseline'|'codegraph'} mode - * @param {import('./token-benchmark-issues.js').BenchmarkIssue} issue * @param {string} nextjsDir - * @returns {Promise} session metrics + * @returns {object} Agent SDK options */ -async function runSession(mode, issue, nextjsDir) { - // Lazy-load the SDK - const { query } = await import('@anthropic-ai/claude-agent-sdk'); - - const dbPath = path.join(nextjsDir, '.codegraph', 'graph.db'); - const cliPath = path.join(root, 'src', 'cli.js'); - const issuePrompt = makeIssuePrompt(issue); - +function buildSessionOptions(mode, nextjsDir) { const options = { cwd: nextjsDir, model: MODEL, @@ -193,6 +188,8 @@ async function runSession(mode, issue, nextjsDir) { }; if (mode === 'codegraph') { + const dbPath = path.join(nextjsDir, '.codegraph', 'graph.db'); + const cliPath = path.join(root, 'src', 'cli.js'); options.mcpServers = { codegraph: { type: 'stdio', @@ -202,38 +199,31 @@ async function runSession(mode, issue, nextjsDir) { }; } + return options; +} + +/** + * Run a single agent session using the Claude Agent SDK. + * + * @param {'baseline'|'codegraph'} mode + * @param {import('./token-benchmark-issues.js').BenchmarkIssue} issue + * @param {string} nextjsDir + * @returns {Promise} session metrics + */ +async function runSession(mode, issue, nextjsDir) { + // Lazy-load the SDK + const { query } = await import('@anthropic-ai/claude-agent-sdk'); + + const options = buildSessionOptions(mode, nextjsDir); + const issuePrompt = makeIssuePrompt(issue); + const start = performance.now(); const result = await query({ prompt: issuePrompt, options }); const durationMs = Math.round(performance.now() - start); - // Extract metrics from the SDK result - const usage = result.usage || {}; - const inputTokens = usage.input_tokens || usage.inputTokens || 0; - const outputTokens = usage.output_tokens || usage.outputTokens || 0; - const cacheReadInputTokens = - usage.cache_read_input_tokens || usage.cacheReadInputTokens || 0; - const totalCostUsd = usage.total_cost_usd || usage.totalCostUsd || 0; - const numTurns = result.num_turns || result.numTurns || 0; - - // Count tool calls by type + const usage = extractUsageMetrics(result); const messages = result.messages || []; - const toolCalls = {}; - let uniqueFilesRead = new Set(); - - for (const msg of messages) { - if (msg.role !== 'assistant') continue; - const blocks = Array.isArray(msg.content) ? msg.content : []; - for (const block of blocks) { - if (block.type === 'tool_use') { - const name = block.name || 'unknown'; - toolCalls[name] = (toolCalls[name] || 0) + 1; - // Track unique files read - if (name === 'Read' && block.input?.file_path) { - uniqueFilesRead.add(block.input.file_path); - } - } - } - } + const { toolCalls, uniqueFilesRead } = tallyToolCalls(messages); // Extract identified files from agent output const agentOutput = extractAgentOutput(messages); @@ -241,14 +231,10 @@ async function runSession(mode, issue, nextjsDir) { const validation = validateResult(issue.id, filesIdentified); return { - inputTokens, - outputTokens, - cacheReadInputTokens, - totalCostUsd: round2(totalCostUsd), - numTurns, + ...usage, durationMs, toolCalls, - uniqueFilesRead: uniqueFilesRead.size, + uniqueFilesRead, filesIdentified, hitRate: validation.hitRate, matched: validation.matched, diff --git a/tests/unit/session-metrics.test.ts b/tests/unit/session-metrics.test.ts new file mode 100644 index 00000000..f77a713b --- /dev/null +++ b/tests/unit/session-metrics.test.ts @@ -0,0 +1,140 @@ +/** + * Unit tests for scripts/lib/session-metrics.ts + * + * Regression coverage for #1906: `runSession` in scripts/token-benchmark.ts + * exceeded codegraph's cognitive/cyclomatic/maxNesting complexity + * thresholds. These pure helpers were extracted out of it; this suite pins + * their behavior (including the snake_case/camelCase field-fallback + * semantics of `firstTruthy`, which must match `a || b || ... || z`, not + * `??`) so a future refactor can't silently change it. + */ + +import { describe, expect, it } from 'vitest'; +import { + collectToolUseBlocks, + extractUsageMetrics, + firstTruthy, + tallyToolCalls, +} from '../../scripts/lib/session-metrics.js'; + +describe('firstTruthy', () => { + it('returns the first truthy value', () => { + expect(firstTruthy(0, 5, 10)).toBe(5); + expect(firstTruthy(undefined, undefined, 3)).toBe(3); + }); + + it('treats 0 as falsy, matching `||` chain semantics (not `??`)', () => { + expect(firstTruthy(0, 0, 7)).toBe(7); + }); + + it('falls back to the last value when nothing is truthy', () => { + expect(firstTruthy(0, undefined, 0)).toBe(0); + }); +}); + +describe('extractUsageMetrics', () => { + it('prefers snake_case (raw API) field names', () => { + const metrics = extractUsageMetrics({ + usage: { + input_tokens: 100, + output_tokens: 50, + cache_read_input_tokens: 10, + total_cost_usd: 0.1234, + }, + num_turns: 3, + }); + expect(metrics).toEqual({ + inputTokens: 100, + outputTokens: 50, + cacheReadInputTokens: 10, + totalCostUsd: 0.12, + numTurns: 3, + }); + }); + + it('falls back to camelCase (SDK-normalized) field names', () => { + const metrics = extractUsageMetrics({ + usage: { inputTokens: 200, outputTokens: 75, cacheReadInputTokens: 5, totalCostUsd: 0.5 }, + numTurns: 4, + }); + expect(metrics).toEqual({ + inputTokens: 200, + outputTokens: 75, + cacheReadInputTokens: 5, + totalCostUsd: 0.5, + numTurns: 4, + }); + }); + + it('defaults every field to 0 when usage/turns are absent', () => { + expect(extractUsageMetrics({})).toEqual({ + inputTokens: 0, + outputTokens: 0, + cacheReadInputTokens: 0, + totalCostUsd: 0, + numTurns: 0, + }); + }); + + it('rounds totalCostUsd to 2 decimal places', () => { + const metrics = extractUsageMetrics({ usage: { total_cost_usd: 1.23456 } }); + expect(metrics.totalCostUsd).toBe(1.23); + }); +}); + +describe('collectToolUseBlocks', () => { + it('collects only tool_use blocks from assistant messages, in order', () => { + const messages = [ + { role: 'user', content: [{ type: 'tool_use', name: 'Read' }] }, + { + role: 'assistant', + content: [ + { type: 'text', text: 'thinking...' }, + { type: 'tool_use', name: 'Glob' }, + { type: 'tool_use', name: 'Read', input: { file_path: '/a.ts' } }, + ], + }, + { role: 'assistant', content: [{ type: 'tool_use', name: 'Bash' }] }, + ]; + const blocks = collectToolUseBlocks(messages as never); + expect(blocks.map((b) => b.name)).toEqual(['Glob', 'Read', 'Bash']); + }); + + it('tolerates non-array content', () => { + const messages = [{ role: 'assistant', content: 'plain text' }]; + expect(collectToolUseBlocks(messages as never)).toEqual([]); + }); + + it('returns an empty array for no messages', () => { + expect(collectToolUseBlocks([])).toEqual([]); + }); +}); + +describe('tallyToolCalls', () => { + it('counts tool calls by name and dedupes files read', () => { + const messages = [ + { + role: 'assistant', + content: [ + { type: 'tool_use', name: 'Read', input: { file_path: '/a.ts' } }, + { type: 'tool_use', name: 'Read', input: { file_path: '/a.ts' } }, + { type: 'tool_use', name: 'Read', input: { file_path: '/b.ts' } }, + { type: 'tool_use', name: 'Grep' }, + ], + }, + ]; + const { toolCalls, uniqueFilesRead } = tallyToolCalls(messages as never); + expect(toolCalls).toEqual({ Read: 3, Grep: 1 }); + expect(uniqueFilesRead).toBe(2); + }); + + it('names unnamed tool_use blocks "unknown"', () => { + const messages = [{ role: 'assistant', content: [{ type: 'tool_use' }] }]; + const { toolCalls } = tallyToolCalls(messages as never); + expect(toolCalls).toEqual({ unknown: 1 }); + }); + + it('returns zero tallies for no messages', () => { + expect(tallyToolCalls([])).toEqual({ toolCalls: {}, uniqueFilesRead: 0 }); + }); +});