diff --git a/.changeset/anthropic-thinking-tokens-metric.md b/.changeset/anthropic-thinking-tokens-metric.md new file mode 100644 index 000000000..b16538632 --- /dev/null +++ b/.changeset/anthropic-thinking-tokens-metric.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(anthropic): Record thinking tokens as metric diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json index 1a4e1fd9c..734032321 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json @@ -646,6 +646,7 @@ } }, "metrics": { + "completion_reasoning_tokens": 42, "completion_tokens": 48, "prompt_cache_creation_1h_tokens": 0, "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt index f9d451a62..461f88e29 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt @@ -573,6 +573,7 @@ span_tree: │ } │ } │ metrics: { + │ "completion_reasoning_tokens": 42, │ "completion_tokens": 48, │ "prompt_cache_creation_1h_tokens": 0, │ "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json index 1a4e1fd9c..734032321 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json @@ -646,6 +646,7 @@ } }, "metrics": { + "completion_reasoning_tokens": 42, "completion_tokens": 48, "prompt_cache_creation_1h_tokens": 0, "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt index f9d451a62..461f88e29 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt @@ -573,6 +573,7 @@ span_tree: │ } │ } │ metrics: { + │ "completion_reasoning_tokens": 42, │ "completion_tokens": 48, │ "prompt_cache_creation_1h_tokens": 0, │ "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts index 28499330c..812053b00 100644 --- a/e2e/scenarios/anthropic-instrumentation/assertions.ts +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -746,7 +746,12 @@ export function defineAnthropicInstrumentationAssertions(options: { time_to_first_token: expect.any(Number), prompt_tokens: expect.any(Number), completion_tokens: expect.any(Number), + completion_reasoning_tokens: expect.any(Number), }); + const metrics = span?.metrics as Record; + expect(metrics.completion_reasoning_tokens).toBeLessThanOrEqual( + metrics.completion_tokens, + ); expect( output?.content?.some((block) => block.type === "thinking"), ).toBe(true); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index ffad58aff..a711c3b7a 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -159,6 +159,42 @@ describe("parseMetricsFromUsage", () => { }); }); + it("should map output_tokens_details.thinking_tokens to completion_reasoning_tokens", () => { + const usage = { + input_tokens: 100, + output_tokens: 80, + output_tokens_details: { + thinking_tokens: 20, + }, + }; + + const result = parseMetricsFromUsageForTest(usage); + + expect(result).toEqual({ + prompt_tokens: 100, + completion_tokens: 80, + completion_reasoning_tokens: 20, + }); + }); + + it("should ignore a null or non-number output token breakdown", () => { + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 80, + output_tokens_details: null, + }), + ).toEqual({ prompt_tokens: 100, completion_tokens: 80 }); + + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 80, + output_tokens_details: { thinking_tokens: "not a number" }, + }), + ).toEqual({ prompt_tokens: 100, completion_tokens: 80 }); + }); + it("should ignore non-number token values", () => { const usage = { input_tokens: "not a number", @@ -283,6 +319,40 @@ describe("aggregateAnthropicStreamChunks", () => { }); }); + it("should carry thinking tokens through from message_delta without double counting", () => { + const chunks = [ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + }, + }, + }, + { + type: "content_block_delta", + delta: { type: "thinking_delta", thinking: "hmm" }, + }, + { + type: "message_delta", + usage: { + output_tokens: 80, + output_tokens_details: { thinking_tokens: 20 }, + }, + }, + ]; + + const result = aggregateAnthropicStreamChunksForTest(chunks); + + expect(result.metrics).toMatchObject({ + prompt_tokens: 10, + completion_tokens: 80, + completion_reasoning_tokens: 20, + tokens: 90, + }); + }); + it("should concatenate multiple text deltas", () => { const chunks = [ { diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 36cad5a5c..e184c927d 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -843,6 +843,14 @@ export function parseMetricsFromUsage( } } + // Thinking tokens are already included in output_tokens. + if (isObject(usage.output_tokens_details)) { + const thinkingTokens = usage.output_tokens_details.thinking_tokens; + if (typeof thinkingTokens === "number") { + metrics.completion_reasoning_tokens = thinkingTokens; + } + } + if (isObject(usage.server_tool_use)) { for (const [name, value] of Object.entries(usage.server_tool_use)) { if (typeof value === "number") { diff --git a/js/src/vendor-sdk-types/anthropic.ts b/js/src/vendor-sdk-types/anthropic.ts index 89cdc644f..cbe9225c3 100644 --- a/js/src/vendor-sdk-types/anthropic.ts +++ b/js/src/vendor-sdk-types/anthropic.ts @@ -195,6 +195,10 @@ export interface AnthropicUsage { cache_read_input_tokens?: number; cache_creation_input_tokens?: number; cache_creation?: AnthropicCacheCreationUsage | null; + output_tokens_details?: { + thinking_tokens?: number; + [key: string]: unknown; + } | null; server_tool_use?: AnthropicServerToolUseUsage; [key: string]: unknown; }