From 1f63555ef178e750f6571d15bee55718b9cc49b4 Mon Sep 17 00:00:00 2001 From: Dylan Pulver Date: Wed, 19 Aug 2026 17:05:38 +0300 Subject: [PATCH 1/4] fix(anthropic): Record output_tokens_details.thinking_tokens as completion_thinking_tokens --- .../anthropic-thinking-tokens-metric.md | 5 ++ ...anthropic-v0-latest-wrapped.span-tree.json | 1 + .../anthropic-v0-latest-wrapped.span-tree.txt | 1 + .../anthropic-v0-latest.span-tree.json | 1 + .../anthropic-v0-latest.span-tree.txt | 1 + .../anthropic-instrumentation/assertions.ts | 12 ++++ .../plugins/anthropic-plugin.test.ts | 72 +++++++++++++++++++ .../plugins/anthropic-plugin.ts | 12 ++++ js/src/vendor-sdk-types/anthropic.ts | 11 +++ 9 files changed, 116 insertions(+) create mode 100644 .changeset/anthropic-thinking-tokens-metric.md diff --git a/.changeset/anthropic-thinking-tokens-metric.md b/.changeset/anthropic-thinking-tokens-metric.md new file mode 100644 index 000000000..74949c1c0 --- /dev/null +++ b/.changeset/anthropic-thinking-tokens-metric.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(anthropic): Record `output_tokens_details.thinking_tokens` as `completion_thinking_tokens` diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json index 1a4e1fd9c..fdcd2d7b8 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json @@ -646,6 +646,7 @@ } }, "metrics": { + "completion_thinking_tokens": 42, "completion_tokens": 48, "prompt_cache_creation_1h_tokens": 0, "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt index f9d451a62..927553ccd 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt @@ -573,6 +573,7 @@ span_tree: │ } │ } │ metrics: { + │ "completion_thinking_tokens": 42, │ "completion_tokens": 48, │ "prompt_cache_creation_1h_tokens": 0, │ "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json index 1a4e1fd9c..fdcd2d7b8 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json @@ -646,6 +646,7 @@ } }, "metrics": { + "completion_thinking_tokens": 42, "completion_tokens": 48, "prompt_cache_creation_1h_tokens": 0, "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt index f9d451a62..927553ccd 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt @@ -573,6 +573,7 @@ span_tree: │ } │ } │ metrics: { + │ "completion_thinking_tokens": 42, │ "completion_tokens": 48, │ "prompt_cache_creation_1h_tokens": 0, │ "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts index 28499330c..ea7bdaa21 100644 --- a/e2e/scenarios/anthropic-instrumentation/assertions.ts +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -747,6 +747,18 @@ export function defineAnthropicInstrumentationAssertions(options: { prompt_tokens: expect.any(Number), completion_tokens: expect.any(Number), }); + // `output_tokens_details.thinking_tokens` only appears in responses + // from newer Anthropic SDK and API versions, so older pinned cassettes + // will not carry it. + const metrics = (span?.metrics ?? {}) as Record; + if ("completion_thinking_tokens" in metrics) { + expect(metrics.completion_thinking_tokens).toEqual( + expect.any(Number), + ); + expect( + metrics.completion_thinking_tokens as number, + ).toBeLessThanOrEqual(metrics.completion_tokens as number); + } expect( output?.content?.some((block) => block.type === "thinking"), ).toBe(true); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index ffad58aff..689cc6aa9 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -159,6 +159,44 @@ describe("parseMetricsFromUsage", () => { }); }); + it("should map output_tokens_details.thinking_tokens to completion_thinking_tokens", () => { + const usage = { + input_tokens: 100, + output_tokens: 80, + output_tokens_details: { + thinking_tokens: 20, + }, + }; + + const result = parseMetricsFromUsageForTest(usage); + + // Thinking tokens are a subset of `output_tokens`, so `completion_tokens` + // stays the inclusive total. + expect(result).toEqual({ + prompt_tokens: 100, + completion_tokens: 80, + completion_thinking_tokens: 20, + }); + }); + + it("should ignore a null or non-number output token breakdown", () => { + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 80, + output_tokens_details: null, + }), + ).toEqual({ prompt_tokens: 100, completion_tokens: 80 }); + + expect( + parseMetricsFromUsageForTest({ + input_tokens: 100, + output_tokens: 80, + output_tokens_details: { thinking_tokens: "not a number" }, + }), + ).toEqual({ prompt_tokens: 100, completion_tokens: 80 }); + }); + it("should ignore non-number token values", () => { const usage = { input_tokens: "not a number", @@ -283,6 +321,40 @@ describe("aggregateAnthropicStreamChunks", () => { }); }); + it("should carry thinking tokens through from the final message_delta usage", () => { + const chunks = [ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + }, + }, + }, + { + type: "content_block_delta", + delta: { type: "thinking_delta", thinking: "hmm" }, + }, + { + type: "message_delta", + usage: { + output_tokens: 80, + output_tokens_details: { thinking_tokens: 20 }, + }, + }, + ]; + + const result = aggregateAnthropicStreamChunksForTest(chunks); + + expect(result.metrics).toMatchObject({ + prompt_tokens: 10, + completion_tokens: 80, + completion_thinking_tokens: 20, + tokens: 90, // thinking tokens are already inside completion_tokens + }); + }); + it("should concatenate multiple text deltas", () => { const chunks = [ { diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 36cad5a5c..4b22d6868 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -843,6 +843,18 @@ export function parseMetricsFromUsage( } } + // Extended and adaptive thinking report the reasoning share of the billed + // output tokens. This mirrors the OpenAI plugin, where + // `output_tokens_details.reasoning_tokens` becomes + // `completion_reasoning_tokens`. The value is a subset of `output_tokens`, + // not additional tokens, so it is not rolled into the total. + if (isObject(usage.output_tokens_details)) { + const thinkingTokens = usage.output_tokens_details.thinking_tokens; + if (typeof thinkingTokens === "number") { + metrics.completion_thinking_tokens = thinkingTokens; + } + } + if (isObject(usage.server_tool_use)) { for (const [name, value] of Object.entries(usage.server_tool_use)) { if (typeof value === "number") { diff --git a/js/src/vendor-sdk-types/anthropic.ts b/js/src/vendor-sdk-types/anthropic.ts index 89cdc644f..df81fa68e 100644 --- a/js/src/vendor-sdk-types/anthropic.ts +++ b/js/src/vendor-sdk-types/anthropic.ts @@ -195,10 +195,21 @@ export interface AnthropicUsage { cache_read_input_tokens?: number; cache_creation_input_tokens?: number; cache_creation?: AnthropicCacheCreationUsage | null; + output_tokens_details?: AnthropicOutputTokensDetails | null; server_tool_use?: AnthropicServerToolUseUsage; [key: string]: unknown; } +/** + * Breakdown of output tokens by category. Reported when extended or adaptive + * thinking is active; `output_tokens` stays the authoritative billing total and + * these are a subset of it. + */ +export interface AnthropicOutputTokensDetails { + thinking_tokens?: number; + [key: string]: unknown; +} + /** * Per-TTL breakdown of cache-write tokens. Only recent Anthropic SDK versions * report this; older ones expose just `cache_creation_input_tokens`. From fb0c825b5b1bf0036d3d1fe2f78376cf14a1801d Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:18:48 +0000 Subject: [PATCH 2/4] use correct metric --- .changeset/anthropic-thinking-tokens-metric.md | 5 ----- .../anthropic-v0-latest-wrapped.span-tree.json | 2 +- .../anthropic-v0-latest-wrapped.span-tree.txt | 2 +- .../anthropic-v0-latest.span-tree.json | 2 +- .../anthropic-v0-latest.span-tree.txt | 2 +- .../anthropic-instrumentation/assertions.ts | 15 ++++----------- .../plugins/anthropic-plugin.test.ts | 6 +++--- .../instrumentation/plugins/anthropic-plugin.ts | 2 +- 8 files changed, 12 insertions(+), 24 deletions(-) delete mode 100644 .changeset/anthropic-thinking-tokens-metric.md diff --git a/.changeset/anthropic-thinking-tokens-metric.md b/.changeset/anthropic-thinking-tokens-metric.md deleted file mode 100644 index 74949c1c0..000000000 --- a/.changeset/anthropic-thinking-tokens-metric.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"braintrust": patch ---- - -fix(anthropic): Record `output_tokens_details.thinking_tokens` as `completion_thinking_tokens` diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json index fdcd2d7b8..734032321 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.json @@ -646,7 +646,7 @@ } }, "metrics": { - "completion_thinking_tokens": 42, + "completion_reasoning_tokens": 42, "completion_tokens": 48, "prompt_cache_creation_1h_tokens": 0, "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt index 927553ccd..461f88e29 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest-wrapped.span-tree.txt @@ -573,7 +573,7 @@ span_tree: │ } │ } │ metrics: { - │ "completion_thinking_tokens": 42, + │ "completion_reasoning_tokens": 42, │ "completion_tokens": 48, │ "prompt_cache_creation_1h_tokens": 0, │ "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json index fdcd2d7b8..734032321 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.json @@ -646,7 +646,7 @@ } }, "metrics": { - "completion_thinking_tokens": 42, + "completion_reasoning_tokens": 42, "completion_tokens": 48, "prompt_cache_creation_1h_tokens": 0, "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt index 927553ccd..461f88e29 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0-latest.span-tree.txt @@ -573,7 +573,7 @@ span_tree: │ } │ } │ metrics: { - │ "completion_thinking_tokens": 42, + │ "completion_reasoning_tokens": 42, │ "completion_tokens": 48, │ "prompt_cache_creation_1h_tokens": 0, │ "prompt_cache_creation_5m_tokens": 0, diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts index ea7bdaa21..0e783ae2c 100644 --- a/e2e/scenarios/anthropic-instrumentation/assertions.ts +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -746,19 +746,12 @@ export function defineAnthropicInstrumentationAssertions(options: { time_to_first_token: expect.any(Number), prompt_tokens: expect.any(Number), completion_tokens: expect.any(Number), + completion_reasoning_tokens: expect.any(Number), }); - // `output_tokens_details.thinking_tokens` only appears in responses - // from newer Anthropic SDK and API versions, so older pinned cassettes - // will not carry it. const metrics = (span?.metrics ?? {}) as Record; - if ("completion_thinking_tokens" in metrics) { - expect(metrics.completion_thinking_tokens).toEqual( - expect.any(Number), - ); - expect( - metrics.completion_thinking_tokens as number, - ).toBeLessThanOrEqual(metrics.completion_tokens as number); - } + expect( + metrics.completion_reasoning_tokens as number, + ).toBeLessThanOrEqual(metrics.completion_tokens as number); expect( output?.content?.some((block) => block.type === "thinking"), ).toBe(true); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index 689cc6aa9..c69f6023f 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -159,7 +159,7 @@ describe("parseMetricsFromUsage", () => { }); }); - it("should map output_tokens_details.thinking_tokens to completion_thinking_tokens", () => { + it("should map output_tokens_details.thinking_tokens to completion_reasoning_tokens", () => { const usage = { input_tokens: 100, output_tokens: 80, @@ -175,7 +175,7 @@ describe("parseMetricsFromUsage", () => { expect(result).toEqual({ prompt_tokens: 100, completion_tokens: 80, - completion_thinking_tokens: 20, + completion_reasoning_tokens: 20, }); }); @@ -350,7 +350,7 @@ describe("aggregateAnthropicStreamChunks", () => { expect(result.metrics).toMatchObject({ prompt_tokens: 10, completion_tokens: 80, - completion_thinking_tokens: 20, + completion_reasoning_tokens: 20, tokens: 90, // thinking tokens are already inside completion_tokens }); }); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 4b22d6868..68b981ceb 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -851,7 +851,7 @@ export function parseMetricsFromUsage( if (isObject(usage.output_tokens_details)) { const thinkingTokens = usage.output_tokens_details.thinking_tokens; if (typeof thinkingTokens === "number") { - metrics.completion_thinking_tokens = thinkingTokens; + metrics.completion_reasoning_tokens = thinkingTokens; } } From 1fcd585dbbc4eff584a96eb7f8223853d38a0b8f Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:32:20 +0000 Subject: [PATCH 3/4] Update PR #2382 --- .changeset/anthropic-thinking-tokens-metric.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/anthropic-thinking-tokens-metric.md diff --git a/.changeset/anthropic-thinking-tokens-metric.md b/.changeset/anthropic-thinking-tokens-metric.md new file mode 100644 index 000000000..b16538632 --- /dev/null +++ b/.changeset/anthropic-thinking-tokens-metric.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(anthropic): Record thinking tokens as metric From bec23ad9f0f5d441ba9d9905e0ebeb9ab42b7829 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:09:07 +0000 Subject: [PATCH 4/4] Update PR #2382 --- .../anthropic-instrumentation/assertions.ts | 8 ++++---- .../plugins/anthropic-plugin.test.ts | 6 ++---- .../instrumentation/plugins/anthropic-plugin.ts | 6 +----- js/src/vendor-sdk-types/anthropic.ts | 15 ++++----------- 4 files changed, 11 insertions(+), 24 deletions(-) diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts index 0e783ae2c..812053b00 100644 --- a/e2e/scenarios/anthropic-instrumentation/assertions.ts +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -748,10 +748,10 @@ export function defineAnthropicInstrumentationAssertions(options: { completion_tokens: expect.any(Number), completion_reasoning_tokens: expect.any(Number), }); - const metrics = (span?.metrics ?? {}) as Record; - expect( - metrics.completion_reasoning_tokens as number, - ).toBeLessThanOrEqual(metrics.completion_tokens as number); + const metrics = span?.metrics as Record; + expect(metrics.completion_reasoning_tokens).toBeLessThanOrEqual( + metrics.completion_tokens, + ); expect( output?.content?.some((block) => block.type === "thinking"), ).toBe(true); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index c69f6023f..a711c3b7a 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -170,8 +170,6 @@ describe("parseMetricsFromUsage", () => { const result = parseMetricsFromUsageForTest(usage); - // Thinking tokens are a subset of `output_tokens`, so `completion_tokens` - // stays the inclusive total. expect(result).toEqual({ prompt_tokens: 100, completion_tokens: 80, @@ -321,7 +319,7 @@ describe("aggregateAnthropicStreamChunks", () => { }); }); - it("should carry thinking tokens through from the final message_delta usage", () => { + it("should carry thinking tokens through from message_delta without double counting", () => { const chunks = [ { type: "message_start", @@ -351,7 +349,7 @@ describe("aggregateAnthropicStreamChunks", () => { prompt_tokens: 10, completion_tokens: 80, completion_reasoning_tokens: 20, - tokens: 90, // thinking tokens are already inside completion_tokens + tokens: 90, }); }); diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index 68b981ceb..e184c927d 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -843,11 +843,7 @@ export function parseMetricsFromUsage( } } - // Extended and adaptive thinking report the reasoning share of the billed - // output tokens. This mirrors the OpenAI plugin, where - // `output_tokens_details.reasoning_tokens` becomes - // `completion_reasoning_tokens`. The value is a subset of `output_tokens`, - // not additional tokens, so it is not rolled into the total. + // Thinking tokens are already included in output_tokens. if (isObject(usage.output_tokens_details)) { const thinkingTokens = usage.output_tokens_details.thinking_tokens; if (typeof thinkingTokens === "number") { diff --git a/js/src/vendor-sdk-types/anthropic.ts b/js/src/vendor-sdk-types/anthropic.ts index df81fa68e..cbe9225c3 100644 --- a/js/src/vendor-sdk-types/anthropic.ts +++ b/js/src/vendor-sdk-types/anthropic.ts @@ -195,21 +195,14 @@ export interface AnthropicUsage { cache_read_input_tokens?: number; cache_creation_input_tokens?: number; cache_creation?: AnthropicCacheCreationUsage | null; - output_tokens_details?: AnthropicOutputTokensDetails | null; + output_tokens_details?: { + thinking_tokens?: number; + [key: string]: unknown; + } | null; server_tool_use?: AnthropicServerToolUseUsage; [key: string]: unknown; } -/** - * Breakdown of output tokens by category. Reported when extended or adaptive - * thinking is active; `output_tokens` stays the authoritative billing total and - * these are a subset of it. - */ -export interface AnthropicOutputTokensDetails { - thinking_tokens?: number; - [key: string]: unknown; -} - /** * Per-TTL breakdown of cache-write tokens. Only recent Anthropic SDK versions * report this; older ones expose just `cache_creation_input_tokens`.