From 25a003358e22eb12a51af0941c4e03bedef74534 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:37:44 +0000 Subject: [PATCH 1/2] ref(eve)!: Bump minimum eve version to `0.34.0` --- e2e/config/pr-comment-scenarios.json | 2 +- .../eve-v0-latest.span-tree.json | 12 +- .../__snapshots__/eve-v0-latest.span-tree.txt | 12 +- .../__snapshots__/eve-v0.span-tree.json | 12 +- .../__snapshots__/eve-v0.span-tree.txt | 12 +- .../eve-instrumentation/agent/agent.ts | 10 +- .../eve-instrumentation/agent/channels/eve.ts | 7 + .../agent/hooks/braintrust.ts | 15 - .../braintrust.ts} | 8 +- .../agent/subagents/researcher/agent.ts | 3 + .../subagents/researcher/hooks/braintrust.ts | 15 - .../researcher/instrumentation/braintrust.ts | 18 + .../eve-instrumentation/cassette-filter.mjs | 114 +- .../eve-instrumentation/package.json | 6 +- .../eve-instrumentation/pnpm-lock.yaml | 89 +- .../eve-instrumentation/scenario.test.ts | 158 +- e2e/scenarios/eve-instrumentation/scenario.ts | 25 +- js/src/exports.ts | 5 +- js/src/instrumentation/index.ts | 5 +- .../plugins/eve-plugin.test.ts | 2931 ++++++---------- js/src/instrumentation/plugins/eve-plugin.ts | 3005 ++++++----------- js/src/vendor-sdk-types/eve.ts | 608 +--- js/src/wrappers/attachment-utils.ts | 20 + 23 files changed, 2485 insertions(+), 4607 deletions(-) create mode 100644 e2e/scenarios/eve-instrumentation/agent/channels/eve.ts delete mode 100644 e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts rename e2e/scenarios/eve-instrumentation/agent/{instrumentation.ts => instrumentation/braintrust.ts} (66%) delete mode 100644 e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts create mode 100644 e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index eb2b3f5f4..a3b6b4c1d 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -631,7 +631,7 @@ "label": "Eve Instrumentation", "metadataScenario": "eve-instrumentation", "variants": [ - { "variantKey": "eve-v0", "label": "v0 pinned" }, + { "variantKey": "eve-v0", "label": "v0.34 pinned" }, { "variantKey": "eve-v0-latest", "label": "v0 latest" } ] }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json index f9869502c..c3eedeee9 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json @@ -53,6 +53,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -233,8 +235,6 @@ "output": "Researcher result: [Eve instrumentation](https://eve.dev/docs/guides/instrumentation)", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -365,6 +365,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -525,6 +527,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -705,8 +709,6 @@ "output": "Researcher result: Eve instrumentation https://eve.dev/docs/guides/instrumentation", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -912,6 +914,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt index 1006edff0..feec9910f 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt @@ -65,6 +65,8 @@ span_tree: │ │ ] │ │ metadata: { │ │ "eve.session_id": "", +│ │ "model": "qwen/qwen3-30b-a3b", +│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -95,8 +97,6 @@ span_tree: │ │ output: "Researcher result: [Eve instrumentation](https://eve.dev/docs/guides/instrumentation)" │ │ metadata: { │ │ "eve.session_id": "", -│ │ "model": "qwen/qwen3-30b-a3b", -│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -347,6 +347,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", +│ "model": "qwen/qwen3-30b-a3b", +│ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -498,6 +500,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", + │ "model": "qwen/qwen3-30b-a3b", + │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -528,8 +532,6 @@ span_tree: │ output: "Researcher result: Eve instrumentation https://eve.dev/docs/guides/instrumentation" │ metadata: { │ "eve.session_id": "", - │ "model": "qwen/qwen3-30b-a3b", - │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -855,6 +857,8 @@ span_tree: ] metadata: { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" } diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json index 6066e92ae..af1d20932 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json @@ -53,6 +53,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -233,8 +235,6 @@ "output": "Researcher result: {\"title\": \"Eve instrumentation\", \"url\": \"https://eve.dev/docs/guides/instrumentation\"}", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -365,6 +365,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -525,6 +527,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -705,8 +709,6 @@ "output": "Researcher result: Eve instrumentation (https://eve.dev/docs/guides/instrumentation)", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -912,6 +914,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt index d86328682..1a17de807 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt @@ -65,6 +65,8 @@ span_tree: │ │ ] │ │ metadata: { │ │ "eve.session_id": "", +│ │ "model": "qwen/qwen3-30b-a3b", +│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -95,8 +97,6 @@ span_tree: │ │ output: "Researcher result: {\"title\": \"Eve instrumentation\", \"url\": \"https://eve.dev/docs/guides/instrumentation\"}" │ │ metadata: { │ │ "eve.session_id": "", -│ │ "model": "qwen/qwen3-30b-a3b", -│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -347,6 +347,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", +│ "model": "qwen/qwen3-30b-a3b", +│ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -498,6 +500,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", + │ "model": "qwen/qwen3-30b-a3b", + │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -528,8 +532,6 @@ span_tree: │ output: "Researcher result: Eve instrumentation (https://eve.dev/docs/guides/instrumentation)" │ metadata: { │ "eve.session_id": "", - │ "model": "qwen/qwen3-30b-a3b", - │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -855,6 +857,8 @@ span_tree: ] metadata: { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" } diff --git a/e2e/scenarios/eve-instrumentation/agent/agent.ts b/e2e/scenarios/eve-instrumentation/agent/agent.ts index 589c856cc..8b518295b 100644 --- a/e2e/scenarios/eve-instrumentation/agent/agent.ts +++ b/e2e/scenarios/eve-instrumentation/agent/agent.ts @@ -18,11 +18,15 @@ const dynamicModel = withReadableReasoning( ); export default defineAgent({ + experimental: { + instrumentationProviders: true, + }, model: defineDynamic({ - fallback: dynamicModel, events: { - "step.started": () => dynamicModel, + "step.started": () => ({ + model: dynamicModel, + modelContextWindowTokens: 8_192, + }), }, }), - modelContextWindowTokens: 8_192, }); diff --git a/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts b/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts new file mode 100644 index 000000000..b74a16488 --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts @@ -0,0 +1,7 @@ +import { none } from "eve/channels/auth"; +import { eveChannel } from "eve/channels/eve"; + +export default eveChannel({ + // This local e2e fixture has no user accounts and only serves the test runner. + auth: none(), +}); diff --git a/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts deleted file mode 100644 index fcac75fde..000000000 --- a/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { braintrustEveHook } from "braintrust"; -import { defineState } from "eve/context"; -import { defineHook } from "eve/hooks"; - -export default defineHook( - braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-instrumentation", - ...(process.env.BRAINTRUST_E2E_RUN_ID - ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } - : {}), - }, - }), -); diff --git a/e2e/scenarios/eve-instrumentation/agent/instrumentation.ts b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts similarity index 66% rename from e2e/scenarios/eve-instrumentation/agent/instrumentation.ts rename to e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts index fff8a737d..9ba5484f2 100644 --- a/e2e/scenarios/eve-instrumentation/agent/instrumentation.ts +++ b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts @@ -1,10 +1,14 @@ import { braintrustEveInstrumentation, initLogger } from "braintrust"; -import { defineState } from "eve/context"; import { defineInstrumentation } from "eve/instrumentation"; export default defineInstrumentation( braintrustEveInstrumentation({ - defineState, + metadata: { + scenario: "eve-instrumentation", + ...(process.env.BRAINTRUST_E2E_RUN_ID + ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } + : {}), + }, setup: ({ agentName }) => { initLogger({ projectName: process.env.BRAINTRUST_E2E_PROJECT_NAME || agentName, diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts index 955d05a88..158110861 100644 --- a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts +++ b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts @@ -11,6 +11,9 @@ const openrouter = createOpenRouter({ export default defineAgent({ description: "Research the Eve instrumentation documentation before the parent reads it.", + experimental: { + instrumentationProviders: true, + }, model: withReadableReasoning( openrouter("qwen/qwen3-30b-a3b", { provider: { diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts deleted file mode 100644 index fcac75fde..000000000 --- a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { braintrustEveHook } from "braintrust"; -import { defineState } from "eve/context"; -import { defineHook } from "eve/hooks"; - -export default defineHook( - braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-instrumentation", - ...(process.env.BRAINTRUST_E2E_RUN_ID - ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } - : {}), - }, - }), -); diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts new file mode 100644 index 000000000..9ba5484f2 --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts @@ -0,0 +1,18 @@ +import { braintrustEveInstrumentation, initLogger } from "braintrust"; +import { defineInstrumentation } from "eve/instrumentation"; + +export default defineInstrumentation( + braintrustEveInstrumentation({ + metadata: { + scenario: "eve-instrumentation", + ...(process.env.BRAINTRUST_E2E_RUN_ID + ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } + : {}), + }, + setup: ({ agentName }) => { + initLogger({ + projectName: process.env.BRAINTRUST_E2E_PROJECT_NAME || agentName, + }); + }, + }), +); diff --git a/e2e/scenarios/eve-instrumentation/cassette-filter.mjs b/e2e/scenarios/eve-instrumentation/cassette-filter.mjs index f73d1a5ef..4066cef31 100644 --- a/e2e/scenarios/eve-instrumentation/cassette-filter.mjs +++ b/e2e/scenarios/eve-instrumentation/cassette-filter.mjs @@ -1,5 +1,117 @@ // @ts-check -export { filter } from "../ai-sdk-instrumentation/cassette-filter.mjs"; +const EVE_BUILTIN_TOOLS = new Set([ + "agent", + "ask_question", + "bash", + "glob", + "grep", + "load_skill", + "read_file", + "todo", + "web_fetch", + "write_file", +]); + +/** @type {import("@braintrust/seinfeld").FilterSpec} */ +export const filter = [ + "default", + { + ignoreBodyFields: [ + "experimental_generateMessageId", + "messageId", + "messages.*.id", + "messages.*.experimental_messageId", + // Eve changed the runtime-owned subagent output-schema declaration + // without changing this fixture's authored tool contract. + "tools.*.function.parameters.properties.outputSchema", + // JSON Schema's draft marker is metadata, not part of the accepted input. + "tools.*.function.parameters.$schema", + ], + normalizeRequest(request) { + if ( + request.body.kind !== "json" || + request.body.value === null || + typeof request.body.value !== "object" || + Array.isArray(request.body.value) + ) { + return request; + } + const body = request.body.value; + if (!Array.isArray(body.tools)) { + return request; + } + const messages = Array.isArray(body.messages) + ? body.messages.map((message) => message) + : body.messages; + if (Array.isArray(messages)) { + for (let index = 0; index < messages.length; index++) { + const message = messages[index]; + if ( + message === null || + typeof message !== "object" || + !("role" in message) || + message.role !== "tool" + ) { + continue; + } + let end = index + 1; + while ( + end < messages.length && + messages[end] !== null && + typeof messages[end] === "object" && + "role" in messages[end] && + messages[end].role === "tool" + ) { + end++; + } + messages.splice( + index, + end - index, + ...messages.slice(index, end).sort((left, right) => { + const leftName = + left !== null && + typeof left === "object" && + "name" in left && + typeof left.name === "string" + ? left.name + : ""; + const rightName = + right !== null && + typeof right === "object" && + "name" in right && + typeof right.name === "string" + ? right.name + : ""; + return leftName.localeCompare(rightName); + }), + ); + index = end - 1; + } + } + return { + ...request, + body: { + kind: "json", + value: { + ...body, + messages, + tools: body.tools.filter( + (tool) => + tool === null || + typeof tool !== "object" || + !("function" in tool) || + tool.function === null || + typeof tool.function !== "object" || + !("name" in tool.function) || + typeof tool.function.name !== "string" || + !EVE_BUILTIN_TOOLS.has(tool.function.name), + ), + }, + }, + }; + }, + }, +]; /** @type {import("@braintrust/seinfeld").RedactionSpec} */ export const redact = [ diff --git a/e2e/scenarios/eve-instrumentation/package.json b/e2e/scenarios/eve-instrumentation/package.json index 202841885..93b789343 100644 --- a/e2e/scenarios/eve-instrumentation/package.json +++ b/e2e/scenarios/eve-instrumentation/package.json @@ -14,9 +14,9 @@ }, "dependencies": { "@openrouter/ai-sdk-provider": "3.0.0", - "ai": "7.0.34", - "eve-v0": "npm:eve@0.22.1", - "eve-v0-latest": "npm:eve@0.27.6", + "ai": "7.0.68", + "eve-v0": "npm:eve@0.34.0", + "eve-v0-latest": "npm:eve@0.39.1", "zod": "4.3.6" } } diff --git a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml index c3cbc645d..e6b942a7a 100644 --- a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml @@ -10,36 +10,36 @@ importers: dependencies: '@openrouter/ai-sdk-provider': specifier: 3.0.0 - version: 3.0.0(ai@7.0.34(zod@4.3.6))(zod@4.3.6) + version: 3.0.0(ai@7.0.68(zod@4.3.6))(zod@4.3.6) ai: - specifier: 7.0.34 - version: 7.0.34(zod@4.3.6) + specifier: 7.0.68 + version: 7.0.68(zod@4.3.6) eve-v0: - specifier: npm:eve@0.22.1 - version: eve@0.22.1(ai@7.0.34(zod@4.3.6)) + specifier: npm:eve@0.34.0 + version: eve@0.34.0(ai@7.0.68(zod@4.3.6)) eve-v0-latest: - specifier: npm:eve@0.27.6 - version: eve@0.27.6(ai@7.0.34(zod@4.3.6)) + specifier: npm:eve@0.39.1 + version: eve@0.39.1(ai@7.0.68(zod@4.3.6)) zod: specifier: 4.3.6 version: 4.3.6 packages: - '@ai-sdk/gateway@4.0.26': - resolution: {integrity: sha512-b/nc3COKtk8IxzgcCi418IoZFky/Bw+Jg8+J0/SBBnu/mOXA4bkIKCu81coEtJAWuH2g5Fvvsa0ar7EpmzNWTw==} + '@ai-sdk/gateway@4.0.54': + resolution: {integrity: sha512-x4fAXDqCtYzB/M5vsIQLYcyrzpJuaRgcIwDSw+lpTMMbgH19fU3ds75GSlHNLzfx6Z5yL4Z9+EMr0GJcqVy9QA==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@5.0.12': - resolution: {integrity: sha512-bbhlOgHeYwrIGheLkM6fhS8hVger8uFPmcOLg+kxc9EFh7y30XYorWhthlYAgpadO3SJhFZrIcEknN7qEqEVvA==} + '@ai-sdk/provider-utils@5.0.27': + resolution: {integrity: sha512-EzAn4pdgG5g0xXtH6lE2zyNmfjDQIDjATkfqzuidEI35g++hh4+07vnjzkT/RmGmIClPZiRj/Q2GMPV2V7mkHw==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider@4.0.3': - resolution: {integrity: sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw==} + '@ai-sdk/provider@4.0.7': + resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==} engines: {node: '>=22'} '@emnapi/core@1.11.1': @@ -178,8 +178,8 @@ packages: '@workflow/serde@4.1.0': resolution: {integrity: sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==} - ai@7.0.34: - resolution: {integrity: sha512-jDqclWYqPGFKcUG4CQiKcBiwi5XqVMqvYy6eJdujVpvE1SDoB6j7RtjrGYz3Bn5B1dzTj1StAlrOebY//WK9/Q==} + ai@7.0.68: + resolution: {integrity: sha512-9QuZOT77wzoxxUC0NcueXhCo3HUHA/1pApIJ9VRyE+9/K+3Innkq4kVhrd9aEnwIviJz2Nga063m+UTsPSdOyw==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -237,13 +237,13 @@ packages: wrangler: optional: true - eve@0.22.1: - resolution: {integrity: sha512-4nqy8SEJpS94fM8sxd0Or+irvf/CmS0qENxouKyyegst3Qtn06xZbuSEoI1p7i4wB6sprr95udXh7Ulct6ABrA==} + eve@0.34.0: + resolution: {integrity: sha512-zgrYIbm8u78Furxfdyzaknf5AxFCzwmTGfLEvgax00BUNXu3eebaos8igjYQyM4q5JAxlWAoZCJuItE6r2S0cw==} engines: {node: '>=24'} hasBin: true peerDependencies: '@opentelemetry/api': ^1.0.0 - ai: ^7.0.0 + ai: ^7.0.58 braintrust: ^3.0.0 just-bash: ^3.0.0 microsandbox: ^0.5.0 @@ -257,13 +257,13 @@ packages: microsandbox: optional: true - eve@0.27.6: - resolution: {integrity: sha512-t2Xr/nlVGc0+Fy9fav8p/CJrwMSQM0zyuOGY1F1XapN0UFssGYBGa6Ml/Q5OnowvWwsGH4m0Tf2QAccayVCvtw==} + eve@0.39.1: + resolution: {integrity: sha512-HTd1A3/GsRKNpJ+6WEW4OEPAhWAId5IZ769hjuY3eJqj3cWeaEvBUDZSjUFjSDit63aqbfur0u7AH1m0JNr7EQ==} engines: {node: '>=24'} hasBin: true peerDependencies: '@opentelemetry/api': ^1.0.0 - ai: ^7.0.34 + ai: ^7.0.58 braintrust: ^3.0.0 just-bash: ^3.0.0 microsandbox: ^0.5.0 @@ -365,6 +365,14 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + + undici@8.9.0: + resolution: {integrity: sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==} + engines: {node: '>=22.19.0'} + unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} @@ -447,22 +455,23 @@ packages: snapshots: - '@ai-sdk/gateway@4.0.26(zod@4.3.6)': + '@ai-sdk/gateway@4.0.54(zod@4.3.6)': dependencies: - '@ai-sdk/provider': 4.0.3 - '@ai-sdk/provider-utils': 5.0.12(zod@4.3.6) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.27(zod@4.3.6) '@vercel/oidc': 3.2.0 zod: 4.3.6 - '@ai-sdk/provider-utils@5.0.12(zod@4.3.6)': + '@ai-sdk/provider-utils@5.0.27(zod@4.3.6)': dependencies: - '@ai-sdk/provider': 4.0.3 + '@ai-sdk/provider': 4.0.7 '@standard-schema/spec': 1.1.0 '@workflow/serde': 4.1.0 eventsource-parser: 3.1.0 + undici: 7.29.0 zod: 4.3.6 - '@ai-sdk/provider@4.0.3': + '@ai-sdk/provider@4.0.7': dependencies: json-schema: 0.4.0 @@ -489,9 +498,9 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true - '@openrouter/ai-sdk-provider@3.0.0(ai@7.0.34(zod@4.3.6))(zod@4.3.6)': + '@openrouter/ai-sdk-provider@3.0.0(ai@7.0.68(zod@4.3.6))(zod@4.3.6)': dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) zod: 4.3.6 '@oxc-project/types@0.138.0': {} @@ -558,11 +567,11 @@ snapshots: '@workflow/serde@4.1.0': {} - ai@7.0.34(zod@4.3.6): + ai@7.0.68(zod@4.3.6): dependencies: - '@ai-sdk/gateway': 4.0.26(zod@4.3.6) - '@ai-sdk/provider': 4.0.3 - '@ai-sdk/provider-utils': 5.0.12(zod@4.3.6) + '@ai-sdk/gateway': 4.0.54(zod@4.3.6) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.27(zod@4.3.6) zod: 4.3.6 consola@3.4.2: {} @@ -580,10 +589,11 @@ snapshots: httpxy: 0.5.4 srvx: 0.11.21 - eve@0.22.1(ai@7.0.34(zod@4.3.6)): + eve@0.34.0(ai@7.0.68(zod@4.3.6)): dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) nitro: 3.0.260610-beta + undici: 8.9.0 transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -624,10 +634,11 @@ snapshots: - xml2js - zephyr-agent - eve@0.27.6(ai@7.0.34(zod@4.3.6)): + eve@0.39.1(ai@7.0.68(zod@4.3.6)): dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) nitro: 3.0.260610-beta + undici: 8.9.0 transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -773,6 +784,10 @@ snapshots: tslib@2.8.1: optional: true + undici@7.29.0: {} + + undici@8.9.0: {} + unenv@2.0.0-rc.24: dependencies: pathe: 2.0.3 diff --git a/e2e/scenarios/eve-instrumentation/scenario.test.ts b/e2e/scenarios/eve-instrumentation/scenario.test.ts index 4d65888c9..2fcd1ad48 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.test.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.test.ts @@ -26,7 +26,7 @@ const eveScenarios = await Promise.all( [ { dependencyName: "eve-v0", - label: "v0 pinned", + label: "v0.34 pinned", variantKey: "eve-v0", }, { @@ -137,12 +137,16 @@ describe.sequential("eve instrumentation variants", () => { "read", secondRoot?.span.id, ); - + const secondChildSearch = findLatestChildSpan( + events, + "search", + secondChildTurn?.span.id, + ); expect(findAllSpans(events, "eve.session")).toEqual([]); expect(turns).toHaveLength(4); expect( turns.filter((turn) => turn.span.parentIds.length === 0), - ).toHaveLength(2); + ).toEqual([root, secondRoot]); expect(new Set(turns.map((turn) => turn.span.rootId)).size).toBe(2); expect(root).toBeDefined(); @@ -155,10 +159,21 @@ describe.sequential("eve instrumentation variants", () => { }); expect(root?.metadata).not.toHaveProperty("model"); expect(root?.metadata).not.toHaveProperty("provider"); - expect(root?.metrics?.completion_tokens).toEqual(expect.any(Number)); - expect(root?.metrics?.prompt_tokens).toEqual(expect.any(Number)); - expect(root?.metrics?.tokens).toEqual(expect.any(Number)); expect(root?.output).toContain("Final answer from read"); + expect(root?.metrics).toMatchObject({ + completion_tokens: steps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: steps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: steps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(steps).toHaveLength(2); expect(steps.map((step) => step.span.type)).toEqual(["llm", "llm"]); @@ -191,11 +206,14 @@ describe.sequential("eve instrumentation variants", () => { expect(step.input[0]).toMatchObject({ role: "system" }); expect(step.metadata).toMatchObject({ "eve.session_id": root?.metadata?.["eve.session_id"], + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", scenario: "eve-instrumentation", testRunId: expect.any(String), }); - expect(step.metadata).not.toHaveProperty("model"); - expect(step.metadata).not.toHaveProperty("provider"); + expect(step.metrics?.completion_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.tokens).toEqual(expect.any(Number)); } expect(researcher).toBeDefined(); @@ -217,14 +235,28 @@ describe.sequential("eve instrumentation variants", () => { expect(childTurn?.span.rootId).toEqual(root?.span.rootId); expect(childTurn?.metadata).toMatchObject({ "eve.session_id": expect.any(String), - model: "qwen/qwen3-30b-a3b", - provider: "openrouter", scenario: "eve-instrumentation", testRunId: expect.any(String), }); + expect(childTurn?.metadata).not.toHaveProperty("model"); + expect(childTurn?.metadata).not.toHaveProperty("provider"); expect(childTurn?.metadata?.["eve.session_id"]).not.toEqual( root?.metadata?.["eve.session_id"], ); + expect(childTurn?.metrics).toMatchObject({ + completion_tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(childSteps).toHaveLength(2); for (const step of childSteps) { @@ -242,6 +274,9 @@ describe.sequential("eve instrumentation variants", () => { scenario: "eve-instrumentation", testRunId: expect.any(String), }); + expect(step.metrics?.completion_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.tokens).toEqual(expect.any(Number)); expect(step.output).toMatchObject([ { message: { @@ -286,74 +321,81 @@ describe.sequential("eve instrumentation variants", () => { expect(secondRoot?.span.rootId).not.toEqual(root?.span.rootId); expect(secondRoot?.metadata).toMatchObject({ "eve.session_id": root?.metadata?.["eve.session_id"], + scenario: "eve-instrumentation", + testRunId: expect.any(String), }); - expect(secondRoot?.metadata).not.toHaveProperty("model"); - expect(secondRoot?.metadata).not.toHaveProperty("provider"); expect(secondRoot?.output).toContain("Final answer from read"); + expect(secondRoot?.metrics).toMatchObject({ + completion_tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(secondSteps).toHaveLength(2); - expect(secondSteps.map((step) => step.span.type)).toEqual([ - "llm", - "llm", - ]); for (const step of secondSteps) { + expect(step.span.parentIds).toEqual([secondRoot?.span.id]); + expect(step.span.rootId).toEqual(secondRoot?.span.rootId); expect(step.metadata).toMatchObject({ "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - scenario: "eve-instrumentation", - testRunId: expect.any(String), + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", }); - expect(step.metadata).not.toHaveProperty("model"); - expect(step.metadata).not.toHaveProperty("provider"); } - expect(secondSteps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - reasoning: [{ content: expect.any(String) }], - tool_calls: [ - { function: { name: "researcher" }, type: "function" }, - { function: { name: "read" }, type: "function" }, - ], - }, - }, - ]); - expect(secondSteps[1]?.output).toMatchObject([ - { - finish_reason: "stop", - message: { - reasoning: [{ content: expect.any(String) }], - }, - }, - ]); - expect(secondResearcher?.span.type).toBe("tool"); - expect(secondResearcher?.span.ended).toBe(true); expect(secondResearcher?.span.parentIds).toEqual([secondRoot?.span.id]); - expect(secondResearcher?.metadata).toMatchObject({ - "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - }); expect(secondChildTurn?.span.parentIds).toEqual([ secondResearcher?.span.id, ]); expect(secondChildTurn?.span.rootId).toEqual(secondRoot?.span.rootId); - expect(secondChildTurn?.metadata).toMatchObject({ - "eve.session_id": expect.any(String), - model: "qwen/qwen3-30b-a3b", - provider: "openrouter", - }); - expect(secondChildTurn?.metadata?.["eve.session_id"]).not.toEqual( - secondRoot?.metadata?.["eve.session_id"], - ); - expect(secondRead?.span.type).toBe("tool"); - expect(secondRead?.span.ended).toBe(true); + expect(secondChildTurn?.span.rootId).not.toEqual(root?.span.rootId); expect(secondRead?.span.parentIds).toEqual([secondRoot?.span.id]); - expect(secondRead?.metadata).toMatchObject({ - "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - }); for (const event of events) { expect(spanInstrumentationName(event)).toBe("eve"); } const rawRows = payloads.flatMap((payload) => payload.rows); + for (const span of [ + ...findAllSpans(events, "eve.step"), + researcher, + childSearch, + read, + secondResearcher, + secondRead, + secondChildSearch, + ]) { + expect(span).toBeDefined(); + expect( + rawRows.filter( + (row) => + row.id === span?.row.id && + Object.prototype.hasOwnProperty.call(row, "input"), + ), + ).toHaveLength(1); + expect( + rawRows.filter( + (row) => + row.id === span?.row.id && + Object.prototype.hasOwnProperty.call(row, "metadata"), + ), + ).toHaveLength(1); + } + for (const turn of turns) { + expect( + rawRows.filter( + (row) => + row.id === turn.row.id && + Object.prototype.hasOwnProperty.call(row, "metadata"), + ), + ).toHaveLength(1); + } for (const step of findAllSpans(events, "eve.step")) { expect( rawRows.filter( diff --git a/e2e/scenarios/eve-instrumentation/scenario.ts b/e2e/scenarios/eve-instrumentation/scenario.ts index 48efc7bea..ca1c92027 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.ts @@ -65,13 +65,10 @@ async function main() { } const body = (await response.json()) as { - continuationToken?: string; sessionId?: string; }; - if (!body.sessionId || !body.continuationToken) { - throw new Error( - `Eve session create did not return a sessionId and continuationToken`, - ); + if (!body.sessionId) { + throw new Error(`Eve session create did not return a sessionId`); } const seenSessionIds = new Set([body.sessionId]); @@ -81,11 +78,13 @@ async function main() { seenSessionIds, "session.waiting", ); + // Eve emits session.waiting just before its durable session snapshot is + // visible to the continuation route. + await new Promise((resolve) => setTimeout(resolve, 1000)); const followUp = await fetch( `${baseUrl}/eve/v1/session/${body.sessionId}`, { body: JSON.stringify({ - continuationToken: body.continuationToken, message: "Run the Braintrust Eve instrumentation e2e scenario again", }), headers: { "content-type": "application/json" }, @@ -213,6 +212,7 @@ async function streamUntil( const decoder = new TextDecoder(); let buffer = ""; let nextIndex = startIndex; + let turnCompleted = false; try { while (true) { const { done, value } = await reader.read(); @@ -228,7 +228,10 @@ async function streamUntil( continue; } const event = JSON.parse(trimmed) as { - data?: { childSessionId?: string; message?: string }; + data?: { + childSessionId?: string; + message?: string; + }; type?: string; }; nextIndex++; @@ -256,7 +259,13 @@ async function streamUntil( ).then(() => undefined), ); } - if (event.type === until) { + if (event.type === "turn.completed") { + turnCompleted = true; + } + if ( + event.type === until && + (until !== "session.waiting" || turnCompleted) + ) { await Promise.all(childStreams); return nextIndex; } diff --git a/js/src/exports.ts b/js/src/exports.ts index 2f4332842..f41d4a6c7 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -196,10 +196,7 @@ export { BraintrustMiddleware, wrapAISDKModel, } from "./wrappers/ai-sdk"; -export { - braintrustEveHook, - braintrustEveInstrumentation, -} from "./instrumentation/plugins/eve-plugin"; +export { braintrustEveInstrumentation } from "./instrumentation/plugins/eve-plugin"; export { collectAnthropicSession } from "./wrappers/anthropic-session-collector"; export { wrapAnthropic } from "./wrappers/anthropic"; export { diff --git a/js/src/instrumentation/index.ts b/js/src/instrumentation/index.ts index 338d62abd..ad2c563d1 100644 --- a/js/src/instrumentation/index.ts +++ b/js/src/instrumentation/index.ts @@ -23,10 +23,7 @@ export { braintrustFlueInstrumentation, braintrustFlueObserver, } from "./plugins/flue-plugin"; -export { - braintrustEveHook, - braintrustEveInstrumentation, -} from "./plugins/eve-plugin"; +export { braintrustEveInstrumentation } from "./plugins/eve-plugin"; // Re-export core types for external instrumentation packages export type { diff --git a/js/src/instrumentation/plugins/eve-plugin.test.ts b/js/src/instrumentation/plugins/eve-plugin.test.ts index bf4323b71..f5db96192 100644 --- a/js/src/instrumentation/plugins/eve-plugin.test.ts +++ b/js/src/instrumentation/plugins/eve-plugin.test.ts @@ -7,46 +7,38 @@ import { it, vi, } from "vitest"; -import { createHash } from "node:crypto"; import * as braintrustExports from "../../exports"; import { configureNode } from "../../node/config"; -import { - _exportsForTestingOnly, - initLogger, - startSpan, - withCurrent, -} from "../../logger"; +import { Attachment, _exportsForTestingOnly, initLogger } from "../../logger"; import * as instrumentationExports from "../index"; -import { braintrustEveHook, braintrustEveInstrumentation } from "./eve-plugin"; +import { braintrustEveInstrumentation } from "./eve-plugin"; import type { - EveHandleMessageStreamEvent, - EveHookContext, + EveInstrumentationAttemptScope, + EveInstrumentationHandlerContext, + EveInstrumentationModelCallCompletedEvent, + EveJsonValue, } from "../../vendor-sdk-types/eve"; -import { mergeRowBatch } from "../../../util/index"; - -function deterministicEveIdForTest(...parts: string[]): string { - return createHash("sha256") - .update(parts.map((part) => `${part.length}:${part}`).join("\0")) - .digest("hex") - .slice(0, 32) - .replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, "$1-$2-$3-$4-$5"); -} -function createFakeDefineState() { - const values = new Map(); +const ATTEMPT_SCOPE: EveInstrumentationAttemptScope = { + attemptId: "attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-root", + stepIndex: 0, + turnId: "turn-0", +}; + +function providerContext( + initial?: EveJsonValue, +): EveInstrumentationHandlerContext { + let value = initial; return { - defineState(name: string, initial: () => T) { - return { - get: () => (values.has(name) ? (values.get(name) as T) : initial()), - update: (fn: (current: T) => T) => { - values.set( - name, - fn(values.has(name) ? (values.get(name) as T) : initial()), - ); - }, - }; + state: { + get: () => value, + set: (next) => { + value = next; + }, }, - values, }; } @@ -56,18 +48,16 @@ try { // Best-effort initialization for test environments. } -describe("braintrustEveHook", () => { +describe("braintrustEveInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; - let defineState: ReturnType["defineState"]; beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); }); beforeEach(() => { - defineState = createFakeDefineState().defineState; backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ projectName: "eve-plugin.test.ts", @@ -80,517 +70,188 @@ describe("braintrustEveHook", () => { _exportsForTestingOnly.clearTestBackgroundLogger(); }); - it("returns an Eve hook definition", () => { - const hook = braintrustEveHook({ defineState }); - - expect(Object.keys(hook)).toEqual(["events"]); - expect(typeof hook.events?.["*"]).toBe("function"); - }); - - it("returns an Eve instrumentation definition", () => { + it("returns an Eve 0.34+ instrumentation provider", () => { const setup = vi.fn(); - const instrumentation = braintrustEveInstrumentation({ - defineState, - setup, - }); - - expect(instrumentation).toMatchObject({ - recordInputs: false, - recordOutputs: false, - setup, - }); - expect(typeof instrumentation.events?.["step.started"]).toBe("function"); - }); - - it("requires Eve's defineState API", () => { - expect(() => braintrustEveHook(undefined as never)).toThrow(); - expect(() => braintrustEveInstrumentation(undefined as never)).toThrow(); + const provider = braintrustEveInstrumentation({ setup }); + + expect(provider.capture).toBe("content"); + expect(provider.setup).toBe(setup); + expect(typeof provider.flush).toBe("function"); + expect(Object.keys(provider.events).sort()).toEqual([ + "action.completed", + "action.failed", + "action.started", + "model.call.completed", + "model.call.failed", + "model.call.started", + "step.attempt.completed", + "step.attempt.failed", + "turn.cancelled", + "turn.completed", + "turn.failed", + "turn.started", + ]); }); - it("exports Eve APIs from root and instrumentation entrypoints", () => { - expect(braintrustExports.braintrustEveHook).toBe(braintrustEveHook); + it("exports only the Eve instrumentation provider API", () => { expect(braintrustExports.braintrustEveInstrumentation).toBe( braintrustEveInstrumentation, ); - expect(instrumentationExports.braintrustEveHook).toBe(braintrustEveHook); expect(instrumentationExports.braintrustEveInstrumentation).toBe( braintrustEveInstrumentation, ); + expect(braintrustExports).not.toHaveProperty("braintrustEveHook"); + expect(instrumentationExports).not.toHaveProperty("braintrustEveHook"); }); - it("captures stable model input without provider options", async () => { - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - const instrumentation = braintrustEveInstrumentation({ defineState }); - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-captured-input" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const modelInput = { - instructions: [ - { - content: "Answer with the relevant Eve instrumentation detail.", - providerOptions: { - openai: { reasoningEncryptedContent: "system-provider-secret" }, - }, - role: "system", - }, - { - content: "Keep the answer concise.", - role: "system", - }, - ], - messages: [ - { - content: [ - { - providerOptions: { - openai: { - itemId: "reasoning-item", - reasoningEncryptedContent: "must-not-be-logged", - }, - }, - text: "Readable reasoning", - type: "reasoning", - }, - { - input: { - providerOptions: { application: "tool-input" }, - reasoningEncryptedContent: "legitimate-tool-input", - }, - providerOptions: { - openai: { reasoningEncryptedContent: "tool-provider-secret" }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-call", - }, - { - output: { - providerOptions: { - openai: { - reasoningEncryptedContent: "output-provider-secret", - }, - }, - type: "json", - value: { - reasoningEncryptedContent: "legitimate-tool-output", - }, - }, - providerOptions: { - openai: { - reasoningEncryptedContent: "result-provider-secret", - }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-result", - }, - ], - providerOptions: { - openai: { reasoningEncryptedContent: "message-provider-secret" }, - }, - role: "assistant", - }, - ], - } as const; - - await emit({ - data: { - runtime: { - agentId: "agent-captured-input", - eveVersion: "0.22.1", - modelId: "dynamic:anthropic/claude-sonnet-5", - }, - }, - type: "session.started", - }); - await emit({ - data: { sequence: 0, turnId: "turn-captured-input" }, - type: "turn.started", - }); - instrumentation.events?.["step.started"]?.({ - modelInput, - session: { id: "session-captured-input" }, - step: { index: 0 }, - turn: { id: "turn-captured-input", sequence: 0 }, - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-captured-input" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Capture the model input.", - sequence: 0, - stepIndex: 0, - turnId: "turn-captured-input", - }, - type: "message.completed", - }); - await emit({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-captured-input", + it("records a turn with LLM and action spans", async () => { + const provider = braintrustEveInstrumentation({ + metadata: { + scenario: "eve-plugin-unit", + testRunId: "test-run-flat-tree", }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-captured-input" }, - type: "turn.completed", }); + const turnContext = providerContext(); + const firstModelContext = providerContext(); + const actionContext = providerContext(); + const secondModelContext = providerContext(); + const scope = ATTEMPT_SCOPE; - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const step = spans.find( - (span) => span.span_attributes?.name === "eve.step", - ); - const session = spans.find( - (span) => span.span_attributes?.name === "eve.session", - ); - const turn = spans.find( - (span) => span.span_attributes?.name === "eve.turn", - ); - for (const span of [turn, step]) { - expect(span?.context?.span_origin).toMatchObject({ - instrumentation: { name: "eve" }, - }); - } - expect(step?.input).toEqual([ + await provider.events["turn.started"]( { - content: "Answer with the relevant Eve instrumentation detail.", - role: "system", + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, + turnContext, + ); + await provider.events["model.call.started"]( { - content: "Keep the answer concise.", - role: "system", + idempotencyKey: "model:attempt-0:0", + input: { + instructions: "You are a research agent.", + messages: [{ content: "Search then read", role: "user" }], + }, + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", + }, + scope, + type: "model.call.started", }, + firstModelContext, + ); + await provider.events["model.call.completed"]( { content: [ + { text: "I should search first.", type: "reasoning" }, { - text: "Readable reasoning", - type: "reasoning", - }, - { - input: { - providerOptions: { application: "tool-input" }, - reasoningEncryptedContent: "legitimate-tool-input", - }, - toolCallId: "call-1", - toolName: "lookup", + callId: "call-search", + input: { query: "Eve instrumentation" }, + toolName: "search", type: "tool-call", }, - { - output: { - type: "json", - value: { - reasoningEncryptedContent: "legitimate-tool-output", - }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-result", - }, ], - role: "assistant", - }, - ]); - expect(JSON.stringify(step?.input)).not.toContain("provider-secret"); - expect(JSON.stringify(step?.input)).not.toContain("must-not-be-logged"); - expect(step?.metadata).toEqual({ - "eve.session_id": "session-captured-input", - }); - expect(session).toBeUndefined(); - expect(turn?.metadata).toEqual({ - "eve.session_id": "session-captured-input", - }); - expect(step?.metadata).not.toHaveProperty("model"); - expect(step?.metadata).not.toHaveProperty("provider"); - expect(turn?.metadata).not.toHaveProperty("model"); - expect(turn?.metadata).not.toHaveProperty("provider"); - expect(fakeEve.values.get("braintrust.eve.tracing")).toMatchObject({ - llmInputs: [], - }); - }); - - it("skips missing or malformed Eve instrumentation state without throwing", async () => { - const instrumentation = braintrustEveInstrumentation({ defineState }); - expect(() => - instrumentation.events?.["step.started"]?.({ bad: true } as never), - ).not.toThrow(); - - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - fakeEve.values.set("braintrust.eve.tracing", { - llmInputs: [{ input: { content: "not an array" }, key: "bad" }], - }); - expect(() => - instrumentation.events?.["step.started"]?.({ - modelInput: { - messages: [{ content: "hello", role: "user" }], + finishReason: "tool-calls", + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.completed", + usage: { + inputTokenDetails: { + cacheReadTokens: 3, + cacheWriteTokens: 2, + }, + inputTokens: 10, + outputTokens: 5, }, - session: { id: "session-malformed-state" }, - step: { index: 0 }, - turn: { id: "turn-malformed-state", sequence: 0 }, - }), - ).not.toThrow(); - - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { session: { id: "session-no-state" } }; - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-no-state" }, - type: "turn.started", }, - ctx, + firstModelContext, ); - await wildcard?.( + await provider.events["action.started"]( { - data: { sequence: 0, stepIndex: 0, turnId: "turn-no-state" }, - type: "step.started", + callId: "call-search", + idempotencyKey: "action:session-root:turn-0:call-search", + input: { query: "Eve instrumentation" }, + name: "search", + scope, + type: "action.started", }, - ctx, + actionContext, ); - await wildcard?.( + await provider.events["action.completed"]( { - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-no-state", + idempotencyKey: "action:session-root:turn-0:call-search", + output: { + output: { hits: ["eve.dev/docs"] }, + type: "result", }, - type: "step.completed", - }, - ctx, - ); - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-no-state" }, - type: "turn.completed", + scope, + type: "action.completed", }, - ctx, - ); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const step = spans.find( - (span) => span.span_attributes?.name === "eve.step", + actionContext, ); - expect(step?.input).toBeUndefined(); - }); - - it("bounds pre-existing durable trace state", async () => { - const fakeEve = createFakeDefineState(); - const oversizedEntryCount = 10_001; - fakeEve.values.set("braintrust.eve.tracing", { - llmInputs: [], - metadata: {}, - spanReferences: Array.from( - { length: oversizedEntryCount }, - (_, index) => ({ - exported: `exported-${index}`, - rootSpanId: `root-${index}`, - rowId: `row-${index}`, - spanId: `span-${index}`, - }), - ), - stepStarts: Array.from({ length: oversizedEntryCount }, (_, index) => ({ - open: false, - ordinal: index, - stepIndex: index, - turnId: `turn-${index}`, - })), - }); - const wildcard = braintrustEveHook({ - defineState: fakeEve.defineState, - }).events?.["*"]; - - await wildcard?.( + await provider.events["model.call.started"]( { - data: { - runtime: { - agentId: "agent-bounded-state", - eveVersion: "0.20.0", - modelId: "openai/gpt-5.4-mini", - }, + idempotencyKey: "model:attempt-1:0", + input: { + instructions: "You are a research agent.", + messages: [ + { content: "Search then read", role: "user" }, + { + content: [ + { + input: { query: "Eve instrumentation" }, + toolCallId: "call-search", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + ], }, - type: "session.started", - }, - { session: { id: "session-bounded-state" } }, - ); - - const state = fakeEve.values.get("braintrust.eve.tracing") as { - spanReferences: unknown[]; - stepStarts: unknown[]; - }; - expect(state.spanReferences).toHaveLength(10_000); - expect(state.stepStarts).toHaveLength(10_000); - }); - - it("does not emit a span for session lifecycle metadata alone", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - - await wildcard?.( - { - data: { - runtime: { - agentId: "agent-session-only", - eveVersion: "0.20.0", - modelId: "openai/gpt-5.4-mini", - }, + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", }, - type: "session.started", + scope, + type: "model.call.started", }, - { session: { id: "session-only" } }, + secondModelContext, ); - - expect(await backgroundLogger.drain()).toEqual([]); - }); - - it("records a flat Eve turn with session model metadata", async () => { - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-flat-tree" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const expectedModelMetadata = { - model: "braintrust-eve-mock", - provider: "eve-mock", - }; - - await emit({ - data: { - runtime: { - agentId: "agent-id", - agentName: "eve-test-agent", - eveVersion: "0.20.0", - modelId: "eve-mock/braintrust-eve-mock", - }, - }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "session.started", - }); - await emit({ - data: { sequence: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "turn.started", - }); - await emit({ - data: { - message: "Search then read", - sequence: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "message.received", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ + await provider.events["model.call.completed"]( + { + content: [ { - callId: "call-search", - input: { query: "Eve instrumentation" }, - kind: "tool-call", - toolName: "search", + text: "Here is the Eve instrumentation guide.", + type: "text", }, ], - sequence: 0, - stepIndex: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "actions.requested", - }); - await emit({ - data: { - error: undefined, - result: { - callId: "call-search", - kind: "tool-result", - output: { hits: ["eve.dev/docs"] }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.050Z" }, - type: "action.result", - }); - await emit({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 0, - turnId: "turn-flat-tree", - usage: { - cacheReadTokens: 3, - cacheWriteTokens: 2, - costUsd: 0.001, - inputTokens: 10, - outputTokens: 5, - }, - }, - meta: { at: "2026-01-01T00:00:00.060Z" }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.070Z" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Here is the Eve instrumentation guide.", - sequence: 0, - stepIndex: 1, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.080Z" }, - type: "message.completed", - }); - await emit({ - data: { finishReason: "stop", - sequence: 0, - stepIndex: 1, - turnId: "turn-flat-tree", + idempotencyKey: "model:attempt-1:0", + scope, + type: "model.call.completed", usage: { inputTokens: 20, outputTokens: 8, }, }, - meta: { at: "2026-01-01T00:00:00.090Z" }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.100Z" }, - type: "turn.completed", - }); + secondModelContext, + ); + await provider.events["turn.completed"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.completed", + }, + turnContext, + ); const spans = (await backgroundLogger.drain()) as Array< Record @@ -598,8 +259,8 @@ describe("braintrustEveHook", () => { const root = spans.find( (span) => span.span_attributes?.name === "eve.turn", ); - const steps = spans.filter((span) => - String(span.span_attributes?.name).startsWith("eve.step"), + const steps = spans.filter( + (span) => span.span_attributes?.name === "eve.step", ); const tool = spans.find((span) => span.span_attributes?.name === "search"); @@ -612,78 +273,58 @@ describe("braintrustEveHook", () => { expect(root).toMatchObject({ input: [{ content: "Search then read", role: "user" }], metadata: { - ...expectedModelMetadata, - "eve.session_id": "session-flat-tree", + "eve.session_id": "session-root", scenario: "eve-plugin-unit", testRunId: "test-run-flat-tree", }, + output: "Here is the Eve instrumentation guide.", metrics: { completion_tokens: 13, - estimated_cost: 0.001, - prompt_cached_tokens: 3, prompt_cache_creation_tokens: 2, + prompt_cached_tokens: 3, prompt_tokens: 30, tokens: 43, }, - output: "Here is the Eve instrumentation guide.", span_attributes: { name: "eve.turn", type: "task", }, - root_span_id: deterministicEveIdForTest( - "eve:root", - "session-flat-tree", - "turn-flat-tree", - ), span_parents: [], }); + expect(root?.root_span_id).not.toBe(root?.span_id); + expect(root?.metadata).not.toHaveProperty("model"); + expect(root?.metadata).not.toHaveProperty("provider"); + expect(steps).toHaveLength(2); - expect(steps.map((span) => span.span_attributes?.name)).toEqual([ - "eve.step", - "eve.step", - ]); - expect(steps.map((span) => span.span_attributes?.type)).toEqual([ - "llm", - "llm", - ]); - expect(steps.map((span) => span.span_parents)).toEqual([ - [root?.span_id], - [root?.span_id], - ]); for (const step of steps) { - expect(step.metadata).toEqual({ - ...expectedModelMetadata, - "eve.session_id": "session-flat-tree", - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", + expect(step).toMatchObject({ + metadata: { + "eve.session_id": "session-root", + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", + scenario: "eve-plugin-unit", + testRunId: "test-run-flat-tree", + }, + span_attributes: { + name: "eve.step", + type: "llm", + }, + span_parents: [root?.span_id], }); } - expect(steps[0]?.input).toBeUndefined(); - expect(steps[1]?.input).toBeUndefined(); - expect(tool).toMatchObject({ - input: { query: "Eve instrumentation" }, - metadata: { - "eve.session_id": "session-flat-tree", - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }, - output: { hits: ["eve.dev/docs"] }, - span_attributes: { - name: "search", - type: "tool", - }, - span_parents: [root?.span_id], - }); - expect(tool?.metadata).not.toHaveProperty("model"); - expect(tool?.metadata).not.toHaveProperty("provider"); + expect(steps[0]?.metadata).not.toHaveProperty("tools"); expect(steps[0]?.output).toMatchObject([ { finish_reason: "tool_calls", message: { + content: null, + reasoning: [{ content: "I should search first." }], tool_calls: [ { function: { - arguments: JSON.stringify({ query: "Eve instrumentation" }), + arguments: JSON.stringify({ + query: "Eve instrumentation", + }), name: "search", }, id: "call-search", @@ -702,1570 +343,1004 @@ describe("braintrustEveHook", () => { }, }, ]); + expect(tool).toMatchObject({ + input: { query: "Eve instrumentation" }, + metadata: { + "eve.session_id": "session-root", + scenario: "eve-plugin-unit", + testRunId: "test-run-flat-tree", + }, + output: { hits: ["eve.dev/docs"] }, + span_attributes: { + name: "search", + type: "tool", + }, + span_parents: [root?.span_id], + }); + expect(tool?.metadata).not.toHaveProperty("model"); + expect(tool?.metadata).not.toHaveProperty("provider"); }); - it("records each user message as a separate turn in one session", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { session: { id: "session-multi-turn" } }; - for (const [sequence, message] of [ - [0, "First user message"], - [1, "Second user message"], - ] as const) { - const turnId = `turn-${sequence}`; - await wildcard?.( - { data: { sequence, turnId }, type: "turn.started" }, - ctx, - ); - await wildcard?.( - { data: { message, sequence, turnId }, type: "message.received" }, - ctx, - ); - await wildcard?.( - { data: { sequence, turnId }, type: "turn.completed" }, - ctx, - ); - } + it("attaches a subagent turn beneath its action span", async () => { + const provider = braintrustEveInstrumentation(); + const parentTurnContext = providerContext(); + const actionContext = providerContext(); + const childTurnContext = providerContext(); + const childModelContext = providerContext(); + const parentScope = ATTEMPT_SCOPE; - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const turns = spans.filter( - (span) => span.span_attributes?.name === "eve.turn", - ); - - expect( - spans.some((span) => span.span_attributes?.name === "eve.session"), - ).toBe(false); - expect(turns).toHaveLength(2); - expect(turns.map((turn) => turn.span_parents)).toEqual([[], []]); - expect(turns.map((turn) => turn.root_span_id)).toEqual([ - deterministicEveIdForTest("eve:root", "session-multi-turn", "turn-0"), - deterministicEveIdForTest("eve:root", "session-multi-turn", "turn-1"), - ]); - expect(turns[0]?.root_span_id).not.toBe(turns[1]?.root_span_id); - expect(turns.map((turn) => turn.input)).toEqual([ - [{ content: "First user message", role: "user" }], - [{ content: "Second user message", role: "user" }], - ]); - }); - - it("merges incremental tool-call batches without reconstructing later LLM inputs", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-incremental-tools" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-incremental-tools" }, - type: "turn.started", - }); - await emit({ - data: { - message: "Search then read", - sequence: 0, - turnId: "turn-incremental-tools", - }, - type: "message.received", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-incremental-tools" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "Eve instrumentation" }, - kind: "tool-call", - toolName: "search", - }, - ], + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - type: "actions.requested", - }); - await emit({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "Updated Eve instrumentation" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", + parentTurnContext, + ); + await provider.events["action.started"]( + { + callId: "call-researcher", + idempotencyKey: "action:session-root:turn-0:call-researcher", + input: { message: "Research Eve" }, + name: "researcher", + scope: parentScope, + type: "action.started", }, - type: "actions.requested", - }); - await emit({ - data: { - actions: [ - { - callId: "call-read", - input: { url: "https://eve.dev/docs/guides/instrumentation" }, - kind: "tool-call", - toolName: "read", + actionContext, + ); + await Promise.all([ + provider.events["turn.started"]( + { + idempotencyKey: "turn:session-child:turn-0", + parentLineage: { + callId: "call-researcher", + sessionId: "session-root", + turnId: "turn-0", }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "actions.requested", - }); - await emit({ - data: { - result: { - callId: "call-search", - kind: "tool-result", - output: { url: "https://eve.dev/docs/guides/instrumentation" }, - toolName: "search", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-child", + turnId: "turn-0", + type: "turn.started", }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "action.result", - }); - await emit({ - data: { - result: { - callId: "call-read", - kind: "tool-result", - output: { excerpt: "Eve hooks expose runtime stream events." }, - toolName: "read", + childTurnContext, + ), + provider.events["model.call.started"]( + { + idempotencyKey: "model:child-attempt-0:0", + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", + }, + scope: { + attemptId: "child-attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "turn-0", + }, + type: "model.call.started", }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "action.result", - }); - await emit({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-incremental-tools" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Done.", - sequence: 0, - stepIndex: 1, - turnId: "turn-incremental-tools", - }, - type: "message.completed", - }); - await emit({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 1, - turnId: "turn-incremental-tools", - }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-incremental-tools" }, - type: "turn.completed", - }); + childModelContext, + ), + ]); const spans = (await backgroundLogger.drain()) as Array< Record >; - const steps = spans.filter((span) => - String(span.span_attributes?.name).startsWith("eve.step"), + const parent = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-root", + ); + const action = spans.find( + (span) => span.span_attributes?.name === "researcher", + ); + const child = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-child", + ); + const childModel = spans.find( + (span) => span.span_attributes?.name === "eve.step", ); - expect(steps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - tool_calls: [ - { - function: { - arguments: JSON.stringify({ - query: "Updated Eve instrumentation", - }), - name: "search", - }, - id: "call-search", - type: "function", - }, - { - function: { - arguments: JSON.stringify({ - url: "https://eve.dev/docs/guides/instrumentation", - }), - name: "read", - }, - id: "call-read", - type: "function", - }, - ], - }, - }, - ]); - expect(steps[1]?.input).toBeUndefined(); + expect(action?.span_parents).toEqual([parent?.span_id]); + expect(child?.span_parents).toEqual([action?.span_id]); + expect(child?.root_span_id).toBe(parent?.root_span_id); + expect(childModel?.span_parents).toEqual([child?.span_id]); + expect(childModel?.root_span_id).toBe(parent?.root_span_id); }); - it("preserves reasoning across tool, text, and structured outputs", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-reasoning-outputs" }, + it("keeps a stable root across nested subagents with different turn ids", async () => { + const provider = braintrustEveInstrumentation(); + const rootContext = providerContext(); + const rootScope = { ...ATTEMPT_SCOPE, turnId: "root-turn" }; + const childScope: EveInstrumentationAttemptScope = { + attemptId: "child-attempt", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "child-turn", }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const turnId = "turn-reasoning-outputs"; - - await emit({ data: { sequence: 0, turnId }, type: "turn.started" }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "I should inspect the available tools.", - sequence: 0, - stepIndex: 0, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - finishReason: "tool-calls", - message: null, - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "message.completed", - }); - await emit({ - data: { - actions: [ - { - callId: "call-reasoning-search", - input: { query: "Eve reasoning" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "actions.requested", - }); - await emit({ - data: { finishReason: "tool-calls", sequence: 0, stepIndex: 0, turnId }, - type: "step.completed", - }); - - await emit({ - data: { sequence: 0, stepIndex: 1, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "The tool returned the relevant guide.", - sequence: 0, - stepIndex: 1, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - reasoning: "I can now answer concisely.", - sequence: 0, - stepIndex: 1, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - finishReason: "stop", - message: "Eve exposes completed reasoning events.", - sequence: 0, - stepIndex: 1, - turnId, - }, - type: "message.completed", - }); - await emit({ - data: { finishReason: "stop", sequence: 0, stepIndex: 1, turnId }, - type: "step.completed", - }); - - await emit({ - data: { sequence: 0, stepIndex: 2, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "The response must match the requested schema.", - sequence: 0, - stepIndex: 2, - turnId, + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:root-turn", + rootSessionId: "session-root", + sequence: 4, + sessionId: "session-root", + turnId: "root-turn", + type: "turn.started", }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - result: { supported: true }, - sequence: 0, - stepIndex: 2, - turnId, + rootContext, + ); + await provider.events["action.started"]( + { + callId: "call-child", + idempotencyKey: "action:session-root:root-turn:call-child", + name: "child", + scope: rootScope, + type: "action.started", }, - type: "result.completed", - }); - await emit({ - data: { finishReason: "stop", sequence: 0, stepIndex: 2, turnId }, - type: "step.completed", - }); - await emit({ data: { sequence: 0, turnId }, type: "turn.completed" }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const steps = spans.filter( - (span) => span.span_attributes?.name === "eve.step", + providerContext(), ); - expect(steps).toHaveLength(3); - expect(steps[0]?.output).toMatchObject([ + await provider.events["turn.started"]( { - finish_reason: "tool_calls", - message: { - content: null, - reasoning: [{ content: "I should inspect the available tools." }], - tool_calls: [ - { - function: { name: "search" }, - id: "call-reasoning-search", - type: "function", - }, - ], + idempotencyKey: "turn:session-child:child-turn", + parentLineage: { + callId: "call-child", + sessionId: "session-root", + turnId: "root-turn", }, + rootSessionId: "session-root", + sequence: 2, + sessionId: "session-child", + turnId: "child-turn", + type: "turn.started", }, - ]); - expect(steps[1]?.output).toMatchObject([ + providerContext(), + ); + await provider.events["action.started"]( { - finish_reason: "stop", - message: { - content: "Eve exposes completed reasoning events.", - reasoning: [ - { content: "The tool returned the relevant guide." }, - { content: "I can now answer concisely." }, - ], - }, + callId: "call-grandchild", + idempotencyKey: "action:session-child:child-turn:call-grandchild", + name: "grandchild", + scope: childScope, + type: "action.started", }, - ]); - expect(steps[2]?.output).toMatchObject([ + providerContext(), + ); + const rootReference = rootContext.state.get(); + if ( + typeof rootReference !== "object" || + rootReference === null || + Array.isArray(rootReference) || + typeof rootReference.rootSpanId !== "string" + ) { + throw new Error("Expected the root turn trace context to be persisted"); + } + const resumedProvider = braintrustEveInstrumentation(); + await resumedProvider.events["turn.started"]( { - finish_reason: "stop", - message: { - content: { supported: true }, - reasoning: [ - { content: "The response must match the requested schema." }, - ], + idempotencyKey: "turn:session-grandchild:grandchild-turn", + parentLineage: { + callId: "call-grandchild", + sessionId: "session-child", + turnId: "child-turn", }, + parentTraceContext: { + spanId: "1111111111111111", + traceFlags: 1, + traceId: rootReference.rootSpanId, + }, + rootSessionId: "session-root", + sequence: 7, + sessionId: "session-grandchild", + turnId: "grandchild-turn", + type: "turn.started", }, + providerContext(), + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const turns = spans.filter( + (span) => span.span_attributes?.name === "eve.turn", + ); + const root = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-root", + ); + const child = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-child", + ); + const grandchild = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-grandchild", + ); + const childAction = spans.find( + (span) => span.span_attributes?.name === "child", + ); + const grandchildAction = spans.find( + (span) => span.span_attributes?.name === "grandchild", + ); + + expect(child?.span_parents).toEqual([childAction?.span_id]); + expect(grandchild?.span_parents).toEqual([grandchildAction?.span_id]); + expect(turns.map((turn) => turn.root_span_id)).toEqual([ + root?.root_span_id, + root?.root_span_id, + root?.root_span_id, ]); }); - it("rehydrates reasoning idempotently across Vercel Workflow steps", async () => { - const eveState = createFakeDefineState(); - const ctx: EveHookContext = { - session: { id: "session-reasoning-replay" }, - }; - const turnId = "turn-reasoning-replay"; - const reasoningEvent = { - data: { - reasoning: "Use the search tool before answering.", - sequence: 0, - stepIndex: 0, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "reasoning.completed", - } as const satisfies EveHandleMessageStreamEvent; - const initialHook = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; + it("creates a separate trace for each top-level turn in one session", async () => { + const provider = braintrustEveInstrumentation(); - await initialHook?.( + await provider.events["turn.started"]( { - data: { sequence: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.000Z" }, + idempotencyKey: "turn:shared-session:turn-0", + rootSessionId: "shared-session", + sequence: 0, + sessionId: "shared-session", + turnId: "turn-0", type: "turn.started", }, - ctx, + providerContext(), ); - await initialHook?.( + await provider.events["turn.started"]( { - data: { sequence: 0, stepIndex: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", + idempotencyKey: "turn:shared-session:turn-1", + rootSessionId: "shared-session", + sequence: 1, + sessionId: "shared-session", + turnId: "turn-1", + type: "turn.started", }, - ctx, + providerContext(), ); - await initialHook?.(reasoningEvent, ctx); - const initialWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } + const turns = (await backgroundLogger.drain()) as Array< + Record >; - expect( - initialWrites - .filter((span) => span.span_attributes?.name) - .every( - (span) => span.context?.span_origin?.instrumentation?.name === "eve", - ), - ).toBe(true); - for (const [key, value] of eveState.values) { - eveState.values.set(key, JSON.parse(JSON.stringify(value))); - } + expect(turns).toHaveLength(2); + expect(turns.map((turn) => turn.metadata?.["eve.session_id"])).toEqual([ + "shared-session", + "shared-session", + ]); + expect(new Set(turns.map((turn) => turn.root_span_id)).size).toBe(2); + }); - const resumedHook = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - await resumedHook?.( + it("preserves a durable subagent turn across provider instances", async () => { + const firstProvider = braintrustEveInstrumentation(); + const parentTurnContext = providerContext(); + const childTurnContext = providerContext(); + + await firstProvider.events["turn.started"]( { - data: { sequence: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.000Z" }, + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", type: "turn.started", }, - ctx, + parentTurnContext, ); - await resumedHook?.( + await firstProvider.events["action.started"]( { - data: { sequence: 0, stepIndex: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", + callId: "call-researcher", + idempotencyKey: "action:session-root:turn-0:call-researcher", + name: "researcher", + scope: ATTEMPT_SCOPE, + type: "action.started", }, - ctx, + providerContext(), ); - const resumedStartWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - await resumedHook?.(reasoningEvent, ctx); - await resumedHook?.( + await firstProvider.events["turn.started"]( { - data: { - finishReason: "tool-calls", - message: null, - sequence: 0, - stepIndex: 0, - turnId, + idempotencyKey: "turn:session-child:turn-0", + parentLineage: { + callId: "call-researcher", + sessionId: "session-root", + turnId: "turn-0", }, - type: "message.completed", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-child", + turnId: "turn-0", + type: "turn.started", }, - ctx, + childTurnContext, ); - await resumedHook?.( + + const resumedProvider = braintrustEveInstrumentation(); + const childScope: EveInstrumentationAttemptScope = { + attemptId: "child-attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "turn-0", + }; + const modelContext = providerContext(); + await resumedProvider.events["model.call.started"]( { - data: { - actions: [ - { - callId: "call-replayed-search", - input: { query: "Vercel Workflow reasoning" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId, + idempotencyKey: "model:child-attempt-0:0", + input: { + messages: [{ content: "Continue durably", role: "user" }], }, - type: "actions.requested", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: childScope, + type: "model.call.started", }, - ctx, + modelContext, ); - const resumedIntermediateWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - await resumedHook?.( + await resumedProvider.events["model.call.completed"]( { - data: { finishReason: "tool-calls", sequence: 0, stepIndex: 0, turnId }, - type: "step.completed", + content: [{ text: "Durable answer", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:child-attempt-0:0", + scope: childScope, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 2 }, }, - ctx, + modelContext, ); - await resumedHook?.( - { data: { sequence: 0, turnId }, type: "turn.completed" }, - ctx, + await resumedProvider.events["turn.completed"]( + { + idempotencyKey: "turn:session-child:turn-0", + sessionId: "session-child", + turnId: "turn-0", + type: "turn.completed", + }, + providerContext(childTurnContext.state.get()), ); - const resumedWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } + const spans = (await backgroundLogger.drain()) as Array< + Record >; - const allWrites = [ - ...initialWrites, - ...resumedStartWrites, - ...resumedIntermediateWrites, - ...resumedWrites, - ]; - const stepRowId = deterministicEveIdForTest( - "eve:row:step", - "session-reasoning-replay", - turnId, - "0", + const parent = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-root", ); - expect( - allWrites.filter( - (write) => write.id === stepRowId && write.output !== undefined, - ), - ).toHaveLength(1); - - const spans = mergeRowBatch([...allWrites].reverse()); - const steps = spans.filter( - (span) => span.span_attributes?.name === "eve.step", + const action = spans.find( + (span) => span.span_attributes?.name === "researcher", ); - expect(steps).toHaveLength(1); - expect(steps[0]?.span_id).toBe( - deterministicEveIdForTest( - "eve:step", - "session-reasoning-replay", - turnId, - "0", - ), + const child = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-child", ); - expect(steps[0]?.output?.[0]?.message).toMatchObject({ - content: null, - reasoning: [{ content: "Use the search tool before answering." }], - tool_calls: [ - { - function: { name: "search" }, - id: "call-replayed-search", - type: "function", - }, - ], - }); - expect(steps[0]?.output?.[0]?.message?.reasoning).toHaveLength(1); - expect(eveState.values.get("braintrust.eve.tracing")).toMatchObject({ - reasoningBlocks: [], - stepStarts: [], + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(child).toMatchObject({ + input: [{ content: "Continue durably", role: "user" }], + metrics: { + completion_tokens: 2, + prompt_tokens: 3, + tokens: 5, + }, + output: "Durable answer", + span_parents: [action?.span_id], }); + expect(child?.root_span_id).toBe(parent?.root_span_id); + expect(step?.span_parents).toEqual([child?.span_id]); + expect(step?.root_span_id).toBe(parent?.root_span_id); }); - it("merges late tool results into tool spans closed by turn completion", async () => { - const eveState = createFakeDefineState(); - const wildcard = braintrustEveHook({ - defineState: eveState.defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); + it("resumes spans and turn metrics from Eve operation state", async () => { + const firstProvider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); + const scope = ATTEMPT_SCOPE; - const ctx: EveHookContext = { - session: { id: "session-late-tool-result" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-late-search", - input: { query: "late tool result" }, - kind: "tool-call", - toolName: "search", - }, - ], + await firstProvider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - stepIndex: 0, - turnId: "turn-late-tool-result", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "actions.requested", - }); - await emit({ - data: { sequence: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "turn.completed", - }); - expect(eveState.values.get("braintrust.eve.tracing")).toMatchObject({ - spanReferences: expect.arrayContaining([ - expect.objectContaining({ - exported: expect.any(String), - rootSpanId: deterministicEveIdForTest( - "eve:root", - "session-late-tool-result", - "turn-late-tool-result", - ), - rowId: deterministicEveIdForTest( - "eve:row:tool", - "session-late-tool-result", - "turn-late-tool-result", - "call-late-search", - ), - spanId: deterministicEveIdForTest( - "eve:tool", - "session-late-tool-result", - "turn-late-tool-result", - "call-late-search", - ), - }), - ]), - stepStarts: [], - }); - const initialWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - const resumedWildcard = braintrustEveHook({ - defineState: eveState.defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - }).events?.["*"]; - const flushSpy = vi.spyOn(backgroundLogger, "flush"); - flushSpy.mockClear(); - await resumedWildcard?.( + turnContext, + ); + await firstProvider.events["model.call.started"]( { - data: { - result: { - callId: "call-late-search", - kind: "tool-result", - output: { title: "Late result" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-late-tool-result", + idempotencyKey: "model:attempt-0:0", + input: { + messages: [{ content: "Resume this call", role: "user" }], + }, + model: { + modelId: "gpt-5.4-mini", + provider: "openai.responses", }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "action.result", + scope, + type: "model.call.started", }, - ctx, - ); - expect(flushSpy).not.toHaveBeenCalled(); - - const resumedWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - expect( - [...initialWrites, ...resumedWrites].every( - (span) => span._is_merge === true, - ), - ).toBe(true); - - // The backend may ingest separate workflow uploads out of order. Because - // every write is a merge, a delayed initial write cannot erase the result. - const spans = mergeRowBatch([...resumedWrites, ...initialWrites]); - const turns = spans.filter( - (span) => span.span_attributes?.name === "eve.turn", + modelContext, ); - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(turns).toHaveLength(1); - expect( - spans.filter((span) => span.span_attributes?.name === "search"), - ).toHaveLength(1); - expect(tool).toMatchObject({ - input: { query: "late tool result" }, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", - }, - output: { title: "Late result" }, - span_attributes: { - name: "search", - type: "tool", - }, - span_parents: [turns[0]?.span_id], + const persisted = modelContext.state.get(); + expect(persisted).toMatchObject({ + exported: expect.any(String), + rootSpanId: expect.any(String), + spanId: expect.any(String), }); - expect(tool?.metrics?.end).toEqual(expect.any(Number)); - }); - it("lets action results complete sparse subagent events across workflow steps", async () => { - const eveState = createFakeDefineState(); - const ctx: EveHookContext = { - session: { id: "session-sparse-subagent" }, - }; - const firstWildcard = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - - await firstWildcard?.( + const resumedProvider = braintrustEveInstrumentation(); + await resumedProvider.events["model.call.completed"]( { - data: { sequence: 0, turnId: "turn-sparse-subagent" }, - type: "turn.started", + content: [{ text: "Resumed output", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.completed", + usage: { inputTokens: 4, outputTokens: 2 }, }, - ctx, + providerContext(persisted), ); - await firstWildcard?.( + const secondModelContext = providerContext(); + await resumedProvider.events["model.call.started"]( { - data: { - actions: [ - { - callId: "call-sparse-subagent", - input: { message: "Research Eve" }, - kind: "subagent-call", - subagentName: "researcher", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-sparse-subagent", + idempotencyKey: "model:attempt-0:1", + input: { + messages: [{ content: "Continue this call", role: "user" }], + }, + model: { + modelId: "gpt-5.4-mini", + provider: "openai.responses", }, - type: "actions.requested", + scope: { ...scope, stepIndex: 1 }, + type: "model.call.started", }, - ctx, + secondModelContext, ); - await firstWildcard?.( - { - data: { - callId: "call-sparse-subagent", - sequence: 0, - subagentName: "researcher", - turnId: "turn-sparse-subagent", + const secondPersisted = secondModelContext.state.get(); + expect(secondPersisted).toMatchObject({ + turnMetricContributions: { + "model:attempt-0:0": { + completion_tokens: 2, + prompt_tokens: 4, + tokens: 6, }, - meta: { at: "2026-01-01T00:00:00.100Z" }, - type: "subagent.completed", }, - ctx, - ); - - const resumedWildcard = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - await resumedWildcard?.( + }); + const finalProvider = braintrustEveInstrumentation(); + await finalProvider.events["model.call.completed"]( { - data: { - result: { - callId: "call-sparse-subagent", - kind: "subagent-result", - output: { answer: "Authoritative result" }, - subagentName: "researcher", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-sparse-subagent", - }, - meta: { at: "2026-01-01T00:00:00.500Z" }, - type: "action.result", + content: [{ text: "Final resumed output", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:attempt-0:1", + scope: { ...scope, stepIndex: 1 }, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 1 }, }, - ctx, + providerContext(secondPersisted), ); const spans = (await backgroundLogger.drain()) as Array< Record >; - const subagents = spans.filter( - (span) => span.span_attributes?.name === "researcher", + const steps = spans.filter( + (span) => span.span_attributes?.name === "eve.step", ); - expect(subagents).toHaveLength(1); - expect(subagents[0]).toMatchObject({ - output: { answer: "Authoritative result" }, - metrics: { end: Date.parse("2026-01-01T00:00:00.100Z") / 1000 }, - span_attributes: { type: "tool" }, - }); - }); - - it("evicts tracing state after session completion", async () => { - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-after-session", + expect(steps).toHaveLength(2); + expect(steps[0]).toMatchObject({ + input: [{ content: "Resume this call", role: "user" }], + metrics: { + completion_tokens: 2, + prompt_tokens: 4, + tokens: 6, }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-late-after-session" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-late-after-session" }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-late-after-session" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-after-session", - input: { query: "after session" }, - kind: "tool-call", - toolName: "search", + output: [ + { + finish_reason: "stop", + message: { + content: "Resumed output", + role: "assistant", }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-late-after-session", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "actions.requested", - }); - await emit({ - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "session.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(tool).toMatchObject({ - input: { query: "after session" }, - }); - expect(tool?.metrics?.end).toEqual(expect.any(Number)); - expect(fakeEve.values.get("braintrust.eve.tracing")).toEqual({ - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], - }); - }); - - it("records result-only tool events without invented input", async () => { - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-result-only", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-result-only" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { - result: { - callId: "call-result-only", - kind: "tool-result", - output: { title: "Result only" }, - toolName: "search", }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-result-only", - }, - type: "action.result", - }); - await emit({ - data: { sequence: 0, turnId: "turn-result-only" }, - type: "turn.completed", + ], }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(tool).toMatchObject({ - output: { title: "Result only" }, - span_attributes: { name: "search", type: "tool" }, + const turn = spans.find( + (span) => span.span_attributes?.name === "eve.turn", + ); + expect(turn?.metrics).toMatchObject({ + completion_tokens: 3, + prompt_tokens: 7, + tokens: 10, }); - expect(tool?.input).toBeUndefined(); }); - it("evicts tracing state after session failure", async () => { - const fakeEve = createFakeDefineState(); - const wildcard = braintrustEveHook({ - defineState: fakeEve.defineState, - }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-failed-cleanup" }, - }; - - await wildcard?.( + it("does not double-count replayed model completions", async () => { + const provider = braintrustEveInstrumentation(); + const modelContext = providerContext(); + await provider.events["turn.started"]( { - data: { sequence: 0, turnId: "turn-failed-cleanup" }, + idempotencyKey: "turn:replay-session:turn-0", + rootSessionId: "replay-session", + sequence: 0, + sessionId: "replay-session", + turnId: "turn-0", type: "turn.started", }, - ctx, - ); - await wildcard?.( - { - data: { - sequence: 0, - stepIndex: 0, - turnId: "turn-failed-cleanup", - }, - type: "step.started", - }, - ctx, + providerContext(), ); - await wildcard?.( + const scope = { + ...ATTEMPT_SCOPE, + attemptId: "replay-attempt", + rootSessionId: "replay-session", + sessionId: "replay-session", + }; + await provider.events["model.call.started"]( { - data: { - code: "session_failed", - message: "Session failed", - sessionId: "session-failed-cleanup", - }, - type: "session.failed", + idempotencyKey: "model:replay-attempt:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope, + type: "model.call.started", }, - ctx, + modelContext, ); + const completed: EveInstrumentationModelCallCompletedEvent = { + content: [{ text: "Done", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:replay-attempt:0", + scope, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 2 }, + }; + await provider.events["model.call.completed"](completed, modelContext); + await provider.events["model.call.completed"](completed, modelContext); - expect(fakeEve.values.get("braintrust.eve.tracing")).toEqual({ - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const turn = spans.find( + (span) => span.span_attributes?.name === "eve.turn", + ); + expect(turn?.metrics).toMatchObject({ + completion_tokens: 2, + prompt_tokens: 3, + tokens: 5, }); }); - it("flushes final session events but not ordinary or ignored events", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-selective-flush" }, - }; + it("records model, action, and turn failures as errors", async () => { + const provider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); + const actionContext = providerContext(); + const scope = ATTEMPT_SCOPE; - await wildcard?.( + await provider.events["turn.started"]( { - data: { sequence: 0, turnId: "turn-selective-flush" }, + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", type: "turn.started", }, - ctx, + turnContext, ); - const flushSpy = vi - .spyOn(backgroundLogger, "flush") - .mockResolvedValue(undefined); - - await wildcard?.( + await provider.events["model.call.started"]( { - data: { wait: "next-user-message" }, - type: "session.waiting", + idempotencyKey: "model:attempt-0:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope, + type: "model.call.started", }, - ctx, + modelContext, ); - expect(flushSpy).not.toHaveBeenCalled(); - - await wildcard?.( + await provider.events["model.call.failed"]( { - data: { - finishReason: "stop", - message: null, - sequence: 0, - stepIndex: 0, - turnId: "turn-selective-flush", - }, - type: "message.completed", + error: new Error("model exploded"), + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.failed", }, - ctx, + modelContext, ); - expect(flushSpy).not.toHaveBeenCalled(); - - await wildcard?.( + await provider.events["action.started"]( { - type: "session.completed", + callId: "call-failing", + idempotencyKey: "action:session-root:turn-0:call-failing", + name: "failing-tool", + scope, + type: "action.started", }, - ctx, + actionContext, + ); + await provider.events["action.failed"]( + { + error: new Error("tool exploded"), + errorCode: "TOOL_FAILED", + idempotencyKey: "action:session-root:turn-0:call-failing", + outcome: "failed", + scope, + type: "action.failed", + }, + actionContext, + ); + await provider.events["turn.failed"]( + { + error: new Error("turn exploded"), + idempotencyKey: "turn:session-root:turn-0", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.failed", + }, + turnContext, ); - expect(flushSpy).toHaveBeenCalledTimes(1); - }); - - it("serializes events per session without blocking other sessions", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const sessionA: EveHookContext = { session: { id: "session-queue-a" } }; - const sessionB: EveHookContext = { session: { id: "session-queue-b" } }; - const emitA = (event: EveHandleMessageStreamEvent) => - wildcard?.(event, sessionA); - const emitB = (event: EveHandleMessageStreamEvent) => - wildcard?.(event, sessionB); - - await emitA({ - data: { sequence: 0, turnId: "turn-a" }, - type: "turn.started", - }); - await emitB({ - data: { sequence: 0, turnId: "turn-b" }, - type: "turn.started", - }); - - let releaseFirstFlush: (() => void) | undefined; - const firstFlush = new Promise((resolve) => { - releaseFirstFlush = resolve; - }); - const flushSpy = vi - .spyOn(backgroundLogger, "flush") - .mockImplementationOnce(() => firstFlush) - .mockResolvedValue(undefined); - - const doneA = emitA({ - type: "session.completed", - }); - for (let i = 0; i < 10 && flushSpy.mock.calls.length < 1; i++) { - await new Promise((resolve) => setTimeout(resolve, 0)); - } - expect(flushSpy).toHaveBeenCalledTimes(1); - - let queuedEventFinished = false; - const queuedA = Promise.resolve( - emitA({ - data: { - finishReason: "stop", - message: "queued", - sequence: 0, - stepIndex: 0, - turnId: "turn-a", - }, - type: "message.completed", - }), - ).then(() => { - queuedEventFinished = true; - }); - await Promise.resolve(); - expect(queuedEventFinished).toBe(false); - - await emitB({ - data: { sequence: 0, turnId: "turn-b" }, - type: "turn.completed", - }); - expect(queuedEventFinished).toBe(false); - releaseFirstFlush?.(); - await Promise.all([doneA, queuedA]); - expect(queuedEventFinished).toBe(true); + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + expect( + spans.find((span) => span.span_attributes?.name === "eve.step")?.error, + ).toContain("model exploded"); + expect( + spans.find((span) => span.span_attributes?.name === "failing-tool") + ?.error, + ).toContain("tool exploded"); + expect( + spans.find((span) => span.span_attributes?.name === "eve.turn")?.error, + ).toContain("turn exploded"); }); - it("uses deterministic ids and attaches local subagent turns to their tool span", async () => { - const parentEveState = createFakeDefineState(); - const childEveState = createFakeDefineState(); - const parentWildcard = braintrustEveHook({ - defineState: parentEveState.defineState, - }).events?.["*"]; - const childWildcard = braintrustEveHook({ - defineState: childEveState.defineState, - }).events?.["*"]; - expect(parentWildcard).toBeDefined(); - expect(childWildcard).toBeDefined(); - - const parentCtx: EveHookContext = { - session: { id: "session-parent" }, - }; - const childCtx: EveHookContext = { - session: { - id: "session-child", - parent: { - callId: "call-researcher", - sessionId: "session-parent", - turn: { id: "turn-parent" }, - }, - }, - }; - const emitParent = (event: EveHandleMessageStreamEvent) => - parentWildcard?.(event, parentCtx); - const emitChild = (event: EveHandleMessageStreamEvent) => - childWildcard?.(event, childCtx); + it("projects model input without provider-private fields", async () => { + const provider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); - await emitParent({ - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.started", - }); - await emitParent({ - data: { - message: "Research Eve tracing", + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - turnId: "turn-parent", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - type: "message.received", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - actions: [ - { - callId: "call-researcher", - input: { message: "Find the relevant section" }, - kind: "subagent-call", - name: "researcher", - subagentName: "researcher", + turnContext, + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + instructions: { + content: "Object system instruction", + providerOptions: { encrypted: "instruction-secret" }, + role: "system", }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-parent", - }, - type: "actions.requested", - }); - await emitParent({ - data: { - callId: "call-researcher", - childSessionId: "session-child", - name: "researcher", - sequence: 0, - toolName: "researcher", - turnId: "turn-parent", + messages: [ + { + content: "Hello", + providerOptions: { reasoning_details: "message-secret" }, + role: "user", + }, + { + content: [ + { + providerOptions: { encrypted: "part-secret" }, + text: "Thinking", + type: "reasoning", + }, + { + input: { query: "Eve" }, + providerOptions: { encrypted: "tool-secret" }, + toolCallId: "call-0", + toolName: "search", + type: "tool-call", + }, + { + providerOptions: { encrypted: "custom-secret" }, + type: "custom", + }, + ], + providerOptions: { reasoning_details: "assistant-secret" }, + role: "assistant", + }, + ], + }, + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", }, - type: "subagent.called", - }); + modelContext, + ); - await emitChild({ - data: { sequence: 0, turnId: "turn-child" }, - type: "turn.started", - }); - await emitChild({ - data: { - message: "Find the relevant section", - sequence: 0, - turnId: "turn-child", - }, - type: "message.received", - }); - await emitChild({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-child" }, - type: "step.started", - }); - await emitChild({ - data: { - actions: [ + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(step?.input).toEqual([ + { content: "Object system instruction", role: "system" }, + { content: "Hello", role: "user" }, + { + content: [ + { text: "Thinking", type: "reasoning" }, { - callId: "call-search", - input: { query: "nested eve" }, - kind: "tool-call", + input: { query: "Eve" }, + toolCallId: "call-0", toolName: "search", + type: "tool-call", }, ], - sequence: 0, - stepIndex: 1, - turnId: "turn-child", - }, - type: "actions.requested", - }); - await emitChild({ - data: { - result: { - callId: "call-search", - kind: "tool-result", - output: { title: "Nested Eve" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 1, - turnId: "turn-child", - }, - type: "action.result", - }); - await emitChild({ - data: { - finishReason: "stop", - message: "Child found Nested Eve.", - sequence: 0, - stepIndex: 0, - turnId: "turn-child", - }, - type: "message.completed", - }); - await emitChild({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-child", + role: "assistant", }, - type: "step.completed", - }); - await emitChild({ - data: { sequence: 0, turnId: "turn-child" }, - type: "turn.completed", - }); + ]); + expect(JSON.stringify(step?.input)).not.toContain("providerOptions"); + expect(JSON.stringify(step?.input)).not.toContain("secret"); + }); - await emitParent({ - data: { - callId: "call-researcher", - output: "Child found Nested Eve.", + it("converts inline model and tool-result media to attachments", async () => { + const provider = braintrustEveInstrumentation(); + const actionContext = providerContext(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - status: "completed", - subagentName: "researcher", - turnId: "turn-parent", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - type: "subagent.completed", - }); - await emitParent({ - data: { - result: { - callId: "call-researcher", - kind: "subagent-result", - output: "Child found Nested Eve.", - subagentName: "researcher", + providerContext(), + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + messages: [ + { + content: [ + { + data: "AQID", + filename: "tiny.png", + mediaType: "image/png", + type: "file", + }, + { + image: "DQ4P", + mediaType: "image/png", + type: "image", + }, + { + data: "not valid base64!", + filename: "invalid.png", + mediaType: "image/png", + type: "file", + }, + ], + role: "user", + }, + { + content: [ + { + output: { + type: "content", + value: [ + { + data: "BAUG", + mediaType: "image/png", + type: "image-data", + }, + ], + }, + toolCallId: "call-image", + toolName: "render", + type: "tool-result", + }, + ], + role: "tool", + }, + ], }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-parent", - }, - type: "action.result", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - actions: [ - { - callId: "call-read", - input: { url: "https://eve.dev/docs/guides/instrumentation" }, - kind: "tool-call", - toolName: "read", - }, - ], - sequence: 0, - stepIndex: 1, - turnId: "turn-parent", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", }, - type: "actions.requested", - }); - await emitParent({ - data: { - result: { - callId: "call-read", - kind: "tool-result", - output: { title: "Runtime context" }, - toolName: "read", + providerContext(), + ); + await provider.events["action.started"]( + { + callId: "call-render", + idempotencyKey: "action:session-root:turn-0:call-render", + input: { + data: "data:image/png;base64,BwgJ", + mediaType: "image/png", + type: "file", }, - sequence: 0, - status: "completed", - stepIndex: 1, - turnId: "turn-parent", - }, - type: "action.result", - }); - await emitParent({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 1, - turnId: "turn-parent", - }, - type: "step.completed", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 2, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - finishReason: "stop", - message: "Parent used the child result.", - sequence: 0, - stepIndex: 2, - turnId: "turn-parent", + name: "render", + scope: ATTEMPT_SCOPE, + type: "action.started", }, - type: "message.completed", - }); - await emitParent({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 2, - turnId: "turn-parent", + actionContext, + ); + await provider.events["action.completed"]( + { + idempotencyKey: "action:session-root:turn-0:call-render", + output: { + output: { + type: "content", + value: [ + { + data: new Uint8Array([10, 11, 12]), + mediaType: "image/png", + type: "image-data", + }, + ], + }, + type: "result", + }, + scope: ATTEMPT_SCOPE, + type: "action.completed", }, - type: "step.completed", - }); - await emitParent({ - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.completed", - }); + actionContext, + ); const spans = (await backgroundLogger.drain()) as Array< Record >; - const parentTurnId = deterministicEveIdForTest( - "eve:turn", - "session-parent", - "turn-parent", - ); - const childTurnId = deterministicEveIdForTest( - "eve:turn", - "session-child", - "turn-child", - ); - const subagentSpanId = deterministicEveIdForTest( - "eve:subagent", - "session-parent", - "call-researcher", - ); - const parentTurn = spans.find( - (span) => - span.span_attributes?.name === "eve.turn" && - span.span_id === parentTurnId, - ); - const subagentSpans = spans.filter( - (span) => span.span_attributes?.name === "researcher", - ); - const childTurn = spans.find( - (span) => - span.span_attributes?.name === "eve.turn" && - span.span_id === childTurnId, - ); - const childSearch = spans.find( - (span) => - span.span_attributes?.name === "search" && - span.span_parents?.[0] === childTurnId, - ); - const parentRead = spans.find( - (span) => - span.span_attributes?.name === "read" && - span.span_parents?.[0] === parentTurnId, - ); - const parentSteps = spans.filter( - (span) => - span.span_attributes?.name === "eve.step" && - span.span_parents?.[0] === parentTurnId, - ); - - expect(parentTurn).toBeDefined(); - expect(subagentSpans).toHaveLength(1); - expect(subagentSpans[0]?.span_id).toBe(subagentSpanId); - expect(subagentSpans[0]?.input).toEqual({ - message: "Find the relevant section", - }); - expect(childTurn).toBeDefined(); - expect(childSearch).toBeDefined(); - expect(parentRead).toBeDefined(); - expect(parentSteps).toHaveLength(3); - expect(parentTurn?.span_parents).toEqual([]); - expect(parentTurn?.span_id).toBe(parentTurnId); - expect(parentTurn?.span_id).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, - ); - expect(parentTurn?.root_span_id).toMatch( - /^([0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/, - ); - expect(parentTurn?.root_span_id).toBe( - deterministicEveIdForTest("eve:root", "session-parent", "turn-parent"), - ); - expect(subagentSpans[0]?.span_parents).toEqual([parentTurn?.span_id]); - expect(childTurn?.span_parents).toEqual([subagentSpanId]); - expect(childTurn?.root_span_id).toBe(parentTurn?.root_span_id); - expect(parentTurn?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(subagentSpans[0]?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(childTurn?.metadata).toEqual({ - "eve.session_id": "session-child", - }); - expect(childSearch?.metadata).toEqual({ - "eve.session_id": "session-child", - }); - expect(parentRead?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(childSearch?.span_parents).toEqual([childTurn?.span_id]); - expect(childSearch?.span_id).toBe( - deterministicEveIdForTest( - "eve:tool", - "session-child", - "turn-child", - "call-search", - ), + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", ); - expect(parentRead?.span_id).toBe( - deterministicEveIdForTest( - "eve:tool", - "session-parent", - "turn-parent", - "call-read", - ), + const inputAttachment = step?.input?.[0]?.content?.[0]?.data; + const imageAttachment = step?.input?.[0]?.content?.[1]?.image; + const unconvertedFile = step?.input?.[0]?.content?.[2]?.data; + const outputAttachment = + step?.input?.[1]?.content?.[0]?.output?.value?.[0]?.data; + const action = spans.find( + (span) => span.span_attributes?.name === "render", ); - expect(parentSteps.map((span) => span.span_id)).toEqual([ - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "0", - ), - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "1", - ), - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "2", - ), - ]); - expect(spans.map((span) => span.span_attributes?.name)).toEqual([ - "eve.turn", - "eve.step", - "researcher", - "eve.turn", - "eve.step", - "search", - "eve.step", - "read", - "eve.step", - ]); + const actionInputAttachment = action?.input?.data; + const actionOutputAttachment = action?.output?.value?.[0]?.data; + expect(inputAttachment).toBeInstanceOf(Attachment); + expect(inputAttachment.reference).toMatchObject({ + content_type: "image/png", + filename: "tiny.png", + type: "braintrust_attachment", + }); + expect(imageAttachment).toBeInstanceOf(Attachment); + expect(imageAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(unconvertedFile).toBe("not valid base64!"); + expect(outputAttachment).toBeInstanceOf(Attachment); + expect(outputAttachment.reference).toMatchObject({ + content_type: "image/png", + filename: "attachment.png", + type: "braintrust_attachment", + }); + expect(actionInputAttachment).toBeInstanceOf(Attachment); + expect(actionInputAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(actionOutputAttachment).toBeInstanceOf(Attachment); + expect(actionOutputAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(JSON.stringify(step?.input)).not.toContain("AQID"); + expect(JSON.stringify(step?.input)).not.toContain("BAUG"); + }); - backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); - initLogger({ - projectName: "eve-plugin.test.ts", - projectId: "test-project-id", - }); - const replay = braintrustEveHook({ - defineState: parentEveState.defineState, - }).events?.["*"]; - await replay?.( + it("preserves arbitrary action outputs with type fields", async () => { + const provider = braintrustEveInstrumentation(); + const actionContext = providerContext(); + await provider.events["turn.started"]( { - data: { sequence: 0, turnId: "turn-parent" }, + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", type: "turn.started", }, - parentCtx, + providerContext(), ); - await replay?.( + await provider.events["action.started"]( { - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.completed", + callId: "call-text", + idempotencyKey: "action:session-root:turn-0:call-text", + name: "text-result", + scope: ATTEMPT_SCOPE, + type: "action.started", + }, + actionContext, + ); + await provider.events["action.completed"]( + { + idempotencyKey: "action:session-root:turn-0:call-text", + output: { + output: { text: "preserve me", type: "text" }, + type: "result", + }, + scope: ATTEMPT_SCOPE, + type: "action.completed", }, - parentCtx, + actionContext, ); - const replaySpans = (await backgroundLogger.drain()) as Array< + + const spans = (await backgroundLogger.drain()) as Array< Record >; expect( - replaySpans.find((span) => span.span_id === parentTurn?.span_id), - ).toMatchObject({ _is_merge: true }); + spans.find((span) => span.span_attributes?.name === "text-result") + ?.output, + ).toEqual({ text: "preserve me", type: "text" }); }); - it("does not parent an Eve turn under the active Braintrust span", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); + it.each([ + ["step.attempt.completed", undefined], + ["step.attempt.failed", new Error("attempt exploded")], + ] as const)("closes an open model span on %s", async (type, error) => { + const provider = braintrustEveInstrumentation(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + providerContext(), + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", + }, + providerContext(), + ); - const ctx: EveHookContext = { - session: { id: "session-wrapped" }, - }; - const parent = startSpan({ name: "workflow" }); - await withCurrent(parent, async () => { - await wildcard?.( + if (type === "step.attempt.completed") { + await provider.events[type]( { - data: { sequence: 0, turnId: "turn-wrapped" }, - type: "turn.started", + idempotencyKey: "step:attempt-0", + scope: ATTEMPT_SCOPE, + type, }, - ctx, + providerContext(), ); - await wildcard?.( + } else { + await provider.events[type]( { - data: { sequence: 0, turnId: "turn-wrapped" }, - type: "turn.completed", + error, + idempotencyKey: "step:attempt-0", + scope: ATTEMPT_SCOPE, + type, }, - ctx, + providerContext(), ); - }); - parent.end(); + } const spans = (await backgroundLogger.drain()) as Array< Record >; - const turn = spans.find( - (span) => span.span_attributes?.name === "eve.turn", - ); - expect(turn?.span_id).toBe( - deterministicEveIdForTest("eve:turn", "session-wrapped", "turn-wrapped"), - ); - expect(turn?.span_parents).toEqual([]); - expect(turn?.root_span_id).toBe( - deterministicEveIdForTest("eve:root", "session-wrapped", "turn-wrapped"), + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", ); - expect(turn?.root_span_id).not.toBe(parent.rootSpanId); - }); - - it("does not throw when Eve emits malformed events or failures", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - await expect( - wildcard?.({ bad: true } as never, {} as never), - ).resolves.toBeUndefined(); - await expect( - wildcard?.( - { - data: { - code: "boom", - message: "step failed", - sequence: 0, - stepIndex: 0, - turnId: "turn-missing-session", - }, - type: "step.failed", - }, - {} as never, - ), - ).resolves.toBeUndefined(); - await expect( - wildcard?.( - { - data: { runtime: { modelId: 123 } }, - type: "session.started", - } as never, - { session: { id: "session-malformed-runtime" } }, - ), - ).resolves.toBeUndefined(); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - expect(spans).toEqual([]); + expect(step?.metrics?.end).toEqual(expect.any(Number)); + if (error !== undefined) { + expect(step?.error).toContain("attempt exploded"); + } }); }); diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/plugins/eve-plugin.ts index ef18f8913..8729c6a48 100644 --- a/js/src/instrumentation/plugins/eve-plugin.ts +++ b/js/src/instrumentation/plugins/eve-plugin.ts @@ -1,7 +1,9 @@ import { debugLogger } from "../../debug-logger"; import { + Attachment, NOOP_SPAN, _internalStartSpanWithInitialMerge, + currentLogger, flush, updateSpan, withCurrent, @@ -14,161 +16,105 @@ import { } from "../../span-origin"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; +import { + convertDataToBlob, + getExtensionFromMediaType, + processInputAttachments, +} from "../../wrappers/attachment-utils"; import type { - EveAssistantStepFinishReason, - EveActionResultError, - EveHandleMessageStreamEvent, - EveHookContext, - EveHookDefinition, + EveInstrumentationActionCompletedEvent, + EveInstrumentationActionFailedEvent, + EveInstrumentationActionStartedEvent, + EveInstrumentationContentPart, EveInstrumentationDefinition, + EveInstrumentationHandlerContext, + EveInstrumentationModelCallCompletedEvent, + EveInstrumentationModelCallFailedEvent, + EveInstrumentationModelCallStartedEvent, EveInstrumentationModelInput, - EveInstrumentationStepStartedEventInput, - EveModelMessage, - EveModelMessageContentPart, - EveRuntimeActionRequest, - EveRuntimeActionResult, - EveRuntimeToolCallActionRequest, - EveRuntimeToolResultActionResult, + EveInstrumentationStepAttemptCompletedEvent, + EveInstrumentationStepAttemptFailedEvent, + EveInstrumentationTurnFailedEvent, + EveInstrumentationTurnSettledEvent, + EveInstrumentationTurnStartedEvent, + EveInstrumentationUsage, } from "../../vendor-sdk-types/eve"; -type SpanState = { - metadata: Record; - span: EveSpan; -}; - type EveSpan = Pick; -type EveSpanStartEvent = { - readonly created: string; - readonly metrics: { readonly start: number }; - readonly span_attributes: Record; - readonly span_parents: string[]; -}; - type EveSpanReference = { - readonly endTime?: number; readonly exported: string; readonly rootSpanId: string; - readonly rowId: string; readonly spanId: string; - readonly startEvent?: EveSpanStartEvent; -}; - -type StepState = SpanState & { - input?: unknown; - metrics: Record; - output?: unknown; - reasoning: readonly EveReasoningBlock[]; + readonly turnMetricContributions?: Record>; }; -type TurnState = SpanState & { +type TurnState = { key: string; - metrics: Record; - output?: unknown; - sessionId: string; - stepsByIndex: Map; - turnId: string; -}; - -type ToolState = SpanState & { - endedByTurn?: boolean; - turnKey: string; -}; - -type EveEntityKind = "step" | "subagent" | "tool" | "turn"; - -type EveStateHandle = { - get(): T; - update(fn: (current: T) => T): void; -}; - -type EveDefineState = (name: string, initial: () => T) => EveStateHandle; - -type EveTraceState = { metadata: Record; - reasoningBlocks: readonly (EveReasoningBlock & { key: string })[]; - spanReferences: readonly EveSpanReference[]; - stepStarts: readonly { - ordinal: number; - open: boolean; - stepIndex: number; - turnId: string; - }[]; - llmInputs: readonly { - input: CapturedEveModelInput; - key: string; - }[]; + rootSpanId: string; + rowId: string; + span: EveSpan; + spanId: string; }; -type EveReasoningBlock = { - content: string; - eventAt?: string; -}; +type EveInstrumentationEvent = + | EveInstrumentationActionCompletedEvent + | EveInstrumentationActionFailedEvent + | EveInstrumentationActionStartedEvent + | EveInstrumentationModelCallCompletedEvent + | EveInstrumentationModelCallFailedEvent + | EveInstrumentationModelCallStartedEvent + | EveInstrumentationStepAttemptCompletedEvent + | EveInstrumentationStepAttemptFailedEvent + | EveInstrumentationTurnFailedEvent + | EveInstrumentationTurnSettledEvent + | EveInstrumentationTurnStartedEvent; -const EVE_TRACE_STATE_KEY = "braintrust.eve.tracing"; const MAX_EVE_CACHE_ENTRIES = 10_000; -const MAX_STORED_LLM_INPUTS = 100; -const MAX_STORED_REASONING_BLOCKS = 100; -const MAX_STORED_SPAN_REFERENCES = 10_000; -const MAX_STORED_STEP_STARTS = 10_000; - -type CapturedEveModelMessage = { - content: string | readonly Record[]; - role: EveModelMessage["role"]; -}; - -type CapturedEveModelInput = readonly CapturedEveModelMessage[]; - -/** Manual hook instrumentation for eve runtime stream events. */ -export function braintrustEveHook(options: { - defineState: EveDefineState; - metadata?: Record; -}): EveHookDefinition { - const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); - const bridge = new EveBridge(state); - return { - events: { - "*": async (event: EveHandleMessageStreamEvent, ctx: EveHookContext) => { - await bridge.handle(event, ctx, options.metadata); - }, - }, - }; -} - -/** Eve instrumentation helper for logger setup and durable LLM input capture. */ -export function braintrustEveInstrumentation(options: { - defineState: EveDefineState; - setup?: EveInstrumentationDefinition["setup"]; -}): EveInstrumentationDefinition { - const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); +const eveRootSpanIdsByTurnKey = new LRUCache({ + max: MAX_EVE_CACHE_ENTRIES, +}); + +/** + * Creates an Eve instrumentation provider. + * + * Eve 0.34.0 introduced the provider event API used here. Older Eve hook and + * single-file instrumentation layouts are intentionally unsupported. + */ +export function braintrustEveInstrumentation( + options: { + metadata?: Record; + setup?: EveInstrumentationDefinition["setup"]; + } = {}, +): EveInstrumentationDefinition { + const bridge = new EveBridge(options.metadata); return { + capture: "content", events: { - "step.started": (input: EveInstrumentationStepStartedEventInput) => { - try { - captureEveModelInput(state, input); - } catch (error) { - debugLogger.warn("Error in Eve LLM input capture:", error); - } - }, + "action.completed": (event, context) => bridge.handle(event, context), + "action.failed": (event, context) => bridge.handle(event, context), + "action.started": (event, context) => bridge.handle(event, context), + "model.call.completed": (event, context) => bridge.handle(event, context), + "model.call.failed": (event, context) => bridge.handle(event, context), + "model.call.started": (event, context) => bridge.handle(event, context), + "step.attempt.completed": (event, context) => + bridge.handle(event, context), + "step.attempt.failed": (event, context) => bridge.handle(event, context), + "turn.cancelled": (event, context) => bridge.handle(event, context), + "turn.completed": (event, context) => bridge.handle(event, context), + "turn.failed": (event, context) => bridge.handle(event, context), + "turn.started": (event, context) => bridge.handle(event, context), }, - recordInputs: false, - recordOutputs: false, + flush: () => flush(), setup: options.setup, }; } -function isEveHandleMessageStreamEvent( - event: unknown, -): event is EveHandleMessageStreamEvent { - return isObject(event) && typeof event["type"] === "string"; -} - class ResumedEveSpan implements EveSpan { private endTime: number | undefined; - constructor(private readonly reference: EveSpanReference) { - this.endTime = reference.endTime; - } + constructor(private readonly reference: EveSpanReference) {} get rootSpanId(): string { return this.reference.rootSpanId; @@ -179,16 +125,9 @@ class ResumedEveSpan implements EveSpan { } log(event: Parameters[0]): void { - const metrics = { - ...this.reference.startEvent?.metrics, - ...(this.endTime === undefined ? {} : { end: this.endTime }), - ...event.metrics, - }; updateSpan({ exported: this.reference.exported, - ...this.reference.startEvent, ...event, - ...(Object.keys(metrics).length > 0 ? { metrics } : {}), }); } @@ -202,2109 +141,1061 @@ class ResumedEveSpan implements EveSpan { } class EveBridge { - constructor(private readonly state: EveStateHandle) {} - - private eventQueuesBySession = new Map>(); - private completedToolKeys = new LRUCache({ - max: MAX_EVE_CACHE_ENTRIES, - }); - private toolsByCallKey = new LRUCache({ - max: MAX_EVE_CACHE_ENTRIES, - }); private turnsByKey = new LRUCache({ max: MAX_EVE_CACHE_ENTRIES, }); + private turnsStartingByKey = new Map>(); + private openModelsByAttempt = new Map>(); + private turnMetricContributionsByKey = new Map< + string, + Record> + >(); - private async startEveSpan( - args: Parameters[0], - ): Promise { - const rowId = args?.event?.id; - const reference = - typeof rowId === "string" && - readEveTraceState(this.state).spanReferences.find( - (candidate) => candidate.rowId === rowId, - ); - if (reference) { - return new ResumedEveSpan(reference); - } - - const startTime = args?.startTime ?? getCurrentUnixTimestamp(); - const parentSpanIds = args?.parentSpanIds; - const startEvent: EveSpanStartEvent = { - created: new Date().toISOString(), - metrics: { start: startTime }, - span_attributes: { - ...(args?.name ? { name: args.name } : {}), - ...(args?.type ? { type: args.type } : {}), - ...args?.spanAttributes, - }, - span_parents: parentSpanIds - ? "spanId" in parentSpanIds - ? [parentSpanIds.spanId] - : parentSpanIds.parentSpanIds - : [], - }; - const span = withCurrent(NOOP_SPAN, () => - _internalStartSpanWithInitialMerge( - withSpanInstrumentationName( - { ...args, startTime }, - INSTRUMENTATION_NAMES.EVE, - ), - ), - ); - if (typeof rowId !== "string") { - return span; - } + constructor(private readonly configuredMetadata?: Record) {} + async handle( + event: EveInstrumentationEvent, + context: EveInstrumentationHandlerContext, + ): Promise { try { - const exported = await span.export(); - const reference = { - exported, - rootSpanId: span.rootSpanId, - rowId, - spanId: span.spanId, - startEvent, - }; - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return normalized.spanReferences.some( - (candidate) => candidate.rowId === rowId, - ) - ? normalized - : { - ...normalized, - spanReferences: [...normalized.spanReferences, reference].slice( - -MAX_STORED_SPAN_REFERENCES, - ), - }; - }); + switch (event.type) { + case "turn.started": + await this.getOrStartTurn(event, context); + return; + case "model.call.started": + await this.handleModelCallStarted(event, context); + return; + case "model.call.completed": + await this.handleModelCallCompleted(event, context); + return; + case "model.call.failed": + await this.handleModelCallFailed(event, context); + return; + case "step.attempt.completed": + case "step.attempt.failed": + this.handleStepAttemptTerminal(event); + return; + case "action.started": + await this.handleActionStarted(event, context); + return; + case "action.completed": + await this.handleActionCompleted(event, context); + return; + case "action.failed": + await this.handleActionFailed(event, context); + return; + case "turn.cancelled": + case "turn.completed": + await this.handleTurnSettled(event, context); + return; + case "turn.failed": + await this.handleTurnFailed(event, context); + return; + } } catch (error) { - debugLogger.warn("Error exporting Eve span for resumption:", error); + debugLogger.warn("Error in Eve instrumentation provider:", error); } - return span; } - private async startEveChildSpan( - parent: EveSpan, - args: Parameters[0], - ): Promise { - return await this.startEveSpan({ - ...args, + private async handleModelCallStarted( + event: EveInstrumentationModelCallStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const metadata = { + ...this.metadata(event.scope.sessionId), + ...modelMetadata(event.model.modelId, event.model.provider), + }; + const input = capturedModelInput(event.input); + const ids = await generateEveIds("step", event.idempotencyKey); + const span = await this.startEveSpan(context, { + event: { + id: ids.rowId, + ...(input === undefined ? {} : { input }), + metadata, + }, + name: "eve.step", parentSpanIds: { - rootSpanId: parent.rootSpanId, - spanId: parent.spanId, + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, }, + spanAttributes: { type: SpanTypeAttribute.LLM }, + spanId: ids.spanId, }); - } + const reference = readSpanReference(context.state.get()); + if (reference !== undefined) { + context.state.set({ + ...reference, + turnMetricContributions: + this.turnMetricContributionsByKey.get(turn.key) ?? {}, + }); + } + const openModels = + this.openModelsByAttempt.get(event.scope.attemptId) ?? new Map(); + openModels.set(event.idempotencyKey, span); + this.openModelsByAttempt.set(event.scope.attemptId, openModels); - private stepOrdinal( - event: Extract, - ): number { - let ordinal = 0; - this.state.update((current) => { - const state = normalizeEveTraceState(current); - const previous = state.stepStarts - .filter( - (entry) => - entry.turnId === event.data.turnId && - entry.stepIndex === event.data.stepIndex, - ) - .at(-1); - if (previous?.open) { - ordinal = previous.ordinal; - return state; + if (input !== undefined) { + const initialInput = initialTurnInput(input); + if (initialInput !== undefined) { + this.updateTurn(turn, { input: initialInput }); } - - ordinal = state.stepStarts.filter( - (entry) => entry.turnId === event.data.turnId, - ).length; - return { - ...state, - stepStarts: [ - ...state.stepStarts, - { - open: true, - ordinal, - stepIndex: event.data.stepIndex, - turnId: event.data.turnId, - }, - ].slice(-MAX_STORED_STEP_STARTS), - }; - }); - return ordinal; + } } - private markStepEnded(turnId: string, stepIndex: number): void { - this.state.update((current) => { - const state = normalizeEveTraceState(current); - let index = -1; - for (let i = state.stepStarts.length - 1; i >= 0; i--) { - const entry = state.stepStarts[i]; - if (entry?.turnId === turnId && entry.stepIndex === stepIndex) { - index = i; - break; - } - } - if (index < 0 || !state.stepStarts[index]?.open) { - return state; - } - return { - ...state, - stepStarts: state.stepStarts.map((entry, entryIndex) => - entryIndex === index ? { ...entry, open: false } : entry, - ), - }; - }); + private async handleModelCallCompleted( + event: EveInstrumentationModelCallCompletedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + this.restoreTurnMetricContributions(turn.key, context.state.get()); + const span = await this.resumeModelSpan(event, context, turn); + const metrics = usageMetrics(event.usage); + const output = modelOutput(event.finishReason, event.content); + span.log({ metrics, output }); + span.end(); + this.closeOpenModel(event.scope.attemptId, event.idempotencyKey); + this.updateTurnMetrics(turn, event.idempotencyKey, metrics); + + const finalOutput = finalText(event.content); + if ( + finalOutput !== undefined && + normalizedFinishReason(event.finishReason) !== "tool_calls" + ) { + this.updateTurn(turn, { output: finalOutput }); + } } - async handle( - event: unknown, - ctx: unknown, - hookMetadata?: Record, + private async handleModelCallFailed( + event: EveInstrumentationModelCallFailedEvent, + context: EveInstrumentationHandlerContext, ): Promise { - if (!isEveHandleMessageStreamEvent(event)) { - return; - } - const run = async () => { - try { - if (!(await this.handleEvent(event, ctx, hookMetadata))) { - return; - } - if (event.type === "session.failed") { - const sessionId = event.data.sessionId || sessionIdFromContext(ctx); - await this.flushInstrumentation(); - if (sessionId) { - this.cleanupSession(sessionId); - } - } else if (event.type === "session.completed") { - const sessionId = sessionIdFromContext(ctx); - await this.flushInstrumentation(); - if (sessionId) { - this.cleanupSession(sessionId); - } - } - } catch (error) { - debugLogger.warn("Error in Eve hook instrumentation:", error); - } - }; + const turn = await this.turnForScope(event.scope, context); + const span = await this.resumeModelSpan(event, context, turn); + span.log({ error: toError(event.error, "Eve model call failed") }); + span.end(); + this.closeOpenModel(event.scope.attemptId, event.idempotencyKey); + } - const sessionId = - event.type === "session.failed" - ? event.data.sessionId || sessionIdFromContext(ctx) - : sessionIdFromContext(ctx); - if (!sessionId) { - await run(); + private handleStepAttemptTerminal( + event: + | EveInstrumentationStepAttemptCompletedEvent + | EveInstrumentationStepAttemptFailedEvent, + ): void { + const openModels = this.openModelsByAttempt.get(event.scope.attemptId); + if (openModels === undefined) { return; } - - const previous = this.eventQueuesBySession.get(sessionId); - const queued = previous ? previous.then(run) : run(); - this.eventQueuesBySession.set(sessionId, queued); - try { - await queued; - } finally { - if (this.eventQueuesBySession.get(sessionId) === queued) { - this.eventQueuesBySession.delete(sessionId); + for (const span of openModels.values()) { + if (event.type === "step.attempt.failed") { + span.log({ + error: toError(event.error, "Eve step attempt failed"), + }); } + span.end(); } + this.openModelsByAttempt.delete(event.scope.attemptId); } - private async handleEvent( - event: EveHandleMessageStreamEvent, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - switch (event.type) { - case "session.started": - this.handleSessionStarted(event, ctx, hookMetadata); - return true; - case "turn.started": - await this.handleTurnStarted(event, ctx, hookMetadata); - return true; - case "message.received": - await this.handleMessageReceived(event, ctx, hookMetadata); - return true; - case "step.started": - await this.handleStepStarted(event, ctx, hookMetadata); - return true; - case "reasoning.completed": - this.handleReasoningCompleted(event, ctx); - return true; - case "message.completed": - this.handleMessageCompleted(event, ctx); - return true; - case "result.completed": - this.handleResultCompleted(event, ctx); - return true; - case "actions.requested": - await this.handleActionsRequested(event, ctx, hookMetadata); - return true; - case "action.result": - await this.handleActionResult(event, ctx, hookMetadata); - return true; - case "subagent.called": - await this.handleSubagentCalled(event, ctx, hookMetadata); - return true; - case "subagent.completed": - await this.handleSubagentCompleted(event, ctx, hookMetadata); - return true; - case "step.completed": - this.handleStepCompleted(event, ctx); - return true; - case "step.failed": - this.handleStepFailed(event, ctx); - return true; - case "turn.completed": - this.handleTurnCompleted(event, ctx); - return true; - case "turn.failed": - this.handleTurnFailed(event, ctx); - return true; - case "session.failed": - this.handleSessionFailed(event, ctx); - return true; - case "session.completed": - this.handleSessionCompleted(event, ctx); - return true; - default: - return false; - } + private async handleActionStarted( + event: EveInstrumentationActionStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const metadata = turn.metadata; + const input = processInputAttachments(event.input); + const ids = await generateEveIds( + "action", + event.scope.sessionId, + event.scope.turnId, + event.callId, + ); + await this.startEveSpan(context, { + event: { + id: ids.rowId, + ...(input === undefined ? {} : { input }), + metadata, + }, + name: event.name, + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + spanId: ids.spanId, + }); } - private handleSessionStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; + private async handleActionCompleted( + event: EveInstrumentationActionCompletedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const span = await this.resumeActionSpan(event, context); + if (event.output.type === "error") { + span.log({ + error: toError(event.output.error, "Eve action failed"), + }); + } else if (event.output.output !== undefined) { + span.log({ output: processInputAttachments(event.output.output) }); } + span.end(actionEndArgs(event.acceptedAtMs)); + } - const metadata = { - ...(hookMetadata ?? {}), - ...modelMetadataFromRuntime(event.data.runtime), - }; - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - metadata: { ...normalized.metadata, ...metadata }, - }; + private async handleActionFailed( + event: EveInstrumentationActionFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const span = await this.resumeActionSpan(event, context); + span.log({ + error: toError( + event.error, + event.errorCode === undefined + ? "Eve action " + event.outcome + : event.errorCode + ": Eve action " + event.outcome, + ), }); - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - - turn.metadata = { ...turn.metadata, ...metadata }; - turn.span.log({ metadata: turn.metadata }); - for (const step of turn.stepsByIndex.values()) { - step.metadata = { ...step.metadata, ...metadata }; - step.span.log({ metadata: step.metadata }); - } - } + span.end(actionEndArgs(event.acceptedAtMs)); } - private async handleTurnStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, + private async handleTurnSettled( + event: EveInstrumentationTurnSettledEvent, + context: EveInstrumentationHandlerContext, ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } + const turn = await this.resumeTurn(event, context); + turn.span.end(); + this.turnsByKey.delete(turn.key); + this.turnMetricContributionsByKey.delete(turn.key); + } - const key = turnKey(sessionId, event.data.turnId); - const metadata = { - ...readEveTraceState(this.state).metadata, - ...(hookMetadata ?? {}), - "eve.session_id": sessionId, - }; - const existing = this.turnsByKey.get(key); - if (existing) { - existing.metadata = { ...existing.metadata, ...metadata }; - existing.span.log({ metadata: existing.metadata }); - return; - } + private async handleTurnFailed( + event: EveInstrumentationTurnFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.resumeTurn(event, context); + turn.span.log({ + error: toError(event.error, "Eve turn failed"), + }); + turn.span.end(); + this.turnsByKey.delete(turn.key); + this.turnMetricContributionsByKey.delete(turn.key); + } - const span = await this.startTurnSpan(sessionId, event, ctx, metadata); - span.log({ metadata }); - this.turnsByKey.set(key, { - key, - metadata, - metrics: {}, - sessionId, - span, - stepsByIndex: new Map(), - turnId: event.data.turnId, + private async startTurnSpan( + event: EveInstrumentationTurnStartedEvent, + context: EveInstrumentationHandlerContext, + metadata: Record, + ): Promise { + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + const parentSpanId = + event.parentLineage === undefined + ? undefined + : await deterministicEveId( + "eve:action", + event.parentLineage.sessionId, + event.parentLineage.turnId, + event.parentLineage.callId, + ); + const parentTurn = + event.parentLineage === undefined + ? undefined + : this.turnsByKey.get( + turnKey(event.parentLineage.sessionId, event.parentLineage.turnId), + ); + const parentRootSpanId = + event.parentLineage === undefined + ? undefined + : eveRootSpanIdsByTurnKey.get( + turnKey(event.parentLineage.sessionId, event.parentLineage.turnId), + ); + const rootSpanId = + event.parentLineage === undefined + ? await deterministicEveId("eve:root", event.sessionId, event.turnId) + : (parentTurn?.rootSpanId ?? + parentRootSpanId ?? + (typeof event.parentTraceContext?.traceId === "string" && + event.parentTraceContext.traceId.length > 0 + ? event.parentTraceContext.traceId + : await deterministicEveId( + "eve:root", + event.parentLineage.sessionId, + event.parentLineage.turnId, + ))); + return await this.startEveSpan(context, { + event: { id: ids.rowId, metadata }, + name: "eve.turn", + parentSpanIds: + parentSpanId === undefined + ? { parentSpanIds: [], rootSpanId } + : { rootSpanId, spanId: parentSpanId }, + spanAttributes: { type: SpanTypeAttribute.TASK }, + spanId: ids.spanId, }); } - private async handleMessageReceived( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - if (!turn) { - return; + private async startEveSpan( + context: EveInstrumentationHandlerContext, + args: Parameters[0], + ): Promise { + const reference = readSpanReference(context.state.get()); + if (reference !== undefined) { + return new ResumedEveSpan(reference); } - const input = [{ content: event.data.message, role: "user" }]; - turn.span.log({ input }); + const startTime = args?.startTime ?? getCurrentUnixTimestamp(); + const span = withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { ...args, startTime }, + INSTRUMENTATION_NAMES.EVE, + ), + ), + ); + + try { + const reference: EveSpanReference = { + exported: await span.export(), + rootSpanId: span.rootSpanId, + spanId: span.spanId, + }; + context.state.set(reference); + } catch (error) { + debugLogger.warn("Error exporting Eve span for resumption:", error); + } + return span; } - private async handleStepStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; + private async getOrStartTurn( + event: EveInstrumentationTurnStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(event.sessionId, event.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; } - const existing = turn.stepsByIndex.get(event.data.stepIndex); - if (existing) { - existing.span.log({ - ...(existing.input !== undefined ? { input: existing.input } : {}), - metadata: existing.metadata, - metrics: existing.metrics, - output: existing.output, - }); - const endTime = eventTime(event); - existing.span.end(endTime === undefined ? undefined : { endTime }); - this.markStepEnded(event.data.turnId, event.data.stepIndex); - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; } - const stepOrdinal = this.stepOrdinal(event); - const metadata = { ...turn.metadata }; - const input = consumeCapturedEveModelInput( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - const reasoning = readStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - const output = mergeEveReasoning(undefined, reasoning); - const { rowId: eventId, spanId } = await generateEveIds( - "step", - sessionId, - event.data.turnId, - String(stepOrdinal), - ); - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - ...(input !== undefined ? { input } : {}), + const metadata = this.metadata(event.sessionId); + const start = (async () => { + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + const span = await this.startTurnSpan(event, context, metadata); + const turn = { + key, metadata, + rootSpanId: span.rootSpanId, + rowId: ids.rowId, + span, + spanId: span.spanId, + }; + this.turnsByKey.set(key, turn); + eveRootSpanIdsByTurnKey.set(key, turn.rootSpanId); + return turn; + })(); + this.turnsStartingByKey.set(key, start); + try { + return await start; + } finally { + if (this.turnsStartingByKey.get(key) === start) { + this.turnsStartingByKey.delete(key); + } + } + } + + private async resumeTurn( + event: + | EveInstrumentationTurnSettledEvent + | EveInstrumentationTurnFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(event.sessionId, event.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; + } + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; + } + + const metadata = this.metadata(event.sessionId); + const span = await this.startTurnSpan( + { + idempotencyKey: event.idempotencyKey, + rootSessionId: event.sessionId, + sequence: 0, + sessionId: event.sessionId, + turnId: event.turnId, + type: "turn.started", }, - name: "eve.step", - spanAttributes: { type: SpanTypeAttribute.LLM }, - spanId, - startTime: eventTime(event), - }); - span.log({ - ...(input !== undefined ? { input } : {}), + context, metadata, - }); - - turn.stepsByIndex.set(event.data.stepIndex, { - ...(input !== undefined ? { input } : {}), + ); + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + return { + key, metadata, - metrics: {}, - ...(output !== undefined ? { output } : {}), - reasoning, + rootSpanId: span.rootSpanId, + rowId: ids.rowId, span, - }); + spanId: span.spanId, + }; } - private handleReasoningCompleted( - event: Extract< - EveHandleMessageStreamEvent, - { type: "reasoning.completed" } - >, - ctx: unknown, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; + private async turnForScope( + scope: EveInstrumentationModelCallStartedEvent["scope"], + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(scope.sessionId, scope.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; } - - const reasoning = storeEveReasoning(this.state, sessionId, event); - const step = this.stepForEvent(event, ctx); - if (step) { - step.reasoning = reasoning; - step.output = mergeEveReasoning(step.output, reasoning); + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; } + + const ids = await generateEveIds("turn", scope.sessionId, scope.turnId); + const persistedRootSpanId = readSpanReference( + context.state.get(), + )?.rootSpanId; + const rememberedRootSpanId = eveRootSpanIdsByTurnKey.get(key); + return { + key, + metadata: this.metadata(scope.sessionId), + rootSpanId: + persistedRootSpanId ?? + rememberedRootSpanId ?? + (await deterministicEveId( + "eve:root", + scope.rootSessionId ?? scope.sessionId, + scope.turnId, + )), + rowId: ids.rowId, + span: NOOP_SPAN, + spanId: ids.spanId, + }; } - private handleMessageCompleted( - event: Extract, - ctx: unknown, + private updateTurn( + turn: TurnState, + event: { input?: unknown; output?: unknown }, ): void { - const step = this.stepForEvent(event, ctx); - if (!step) { + currentLogger()?.updateSpan({ + id: turn.rowId, + root_span_id: turn.rootSpanId, + span_id: turn.spanId, + ...event, + }); + } + + private closeOpenModel(attemptId: string, idempotencyKey: string): void { + const openModels = this.openModelsByAttempt.get(attemptId); + if (openModels === undefined) { return; } - - const existingMessage = eveOutputMessage(step.output); - const existingToolCalls = isObject(existingMessage) - ? existingMessage.tool_calls - : undefined; - step.output = mergeEveReasoning( - [ - { - finish_reason: normalizedFinishReason(event.data.finishReason), - index: 0, - message: { - content: event.data.message, - role: "assistant", - ...(Array.isArray(existingToolCalls) - ? { tool_calls: existingToolCalls } - : {}), - }, - }, - ], - step.reasoning, - ); - - const turn = this.turnForEvent(event, ctx); - if (turn && event.data.finishReason !== "tool-calls") { - turn.output = event.data.message; + openModels.delete(idempotencyKey); + if (openModels.size === 0) { + this.openModelsByAttempt.delete(attemptId); } } - private handleResultCompleted( - event: Extract, - ctx: unknown, + private restoreTurnMetricContributions( + turnKey: string, + state: unknown, ): void { - const step = this.stepForEvent(event, ctx); - if (step) { - step.output = mergeEveReasoning( - [ - { - finish_reason: "stop", - index: 0, - message: { - content: event.data.result, - role: "assistant", - }, - }, - ], - step.reasoning, - ); - } - - const turn = this.turnForEvent(event, ctx); - if (turn) { - turn.output = event.data.result; + const persisted = readSpanReference(state)?.turnMetricContributions; + if ( + persisted === undefined || + this.turnMetricContributionsByKey.has(turnKey) + ) { + return; } + this.turnMetricContributionsByKey.set(turnKey, { ...persisted }); } - private async handleActionsRequested( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; + private updateTurnMetrics( + turn: TurnState, + idempotencyKey: string, + metrics: Record, + ): void { + const contributions = { + ...(this.turnMetricContributionsByKey.get(turn.key) ?? {}), + [idempotencyKey]: metrics, + }; + this.turnMetricContributionsByKey.set(turn.key, contributions); + const totals: Record = {}; + for (const contribution of Object.values(contributions)) { + for (const [name, value] of Object.entries(contribution)) { + totals[name] = (totals[name] ?? 0) + value; + } } - - const traceActions = event.data.actions.filter(isTraceableActionRequest); - if (traceActions.length === 0) { + if (Object.keys(totals).length === 0) { return; } - - for (const action of traceActions) { - if (isToolCallAction(action)) { - await this.startRequestedTool(event, turn, sessionId, action); - } else if (isLocalSubagentCallAction(action)) { - await this.startRequestedSubagent(event, turn, sessionId, action); - } - } - - const step = turn.stepsByIndex.get(event.data.stepIndex); - if (!step) { - return; - } - - const toolCallsById = new Map(); - if (Array.isArray(step.output) && isObject(step.output[0])) { - const message = step.output[0]["message"]; - if (isObject(message) && Array.isArray(message["tool_calls"])) { - for (const toolCall of message["tool_calls"]) { - if (isObject(toolCall) && typeof toolCall["id"] === "string") { - toolCallsById.set(toolCall["id"], toolCall); - } - } - } - } - for (const action of traceActions) { - const name = - action.kind === "tool-call" - ? action.toolName - : (action.subagentName ?? action.name ?? "agent"); - toolCallsById.set(action.callId, { - function: { - arguments: JSON.stringify(action.input), - name, - }, - id: action.callId, - type: "function", - }); - } - - step.output = mergeEveReasoning( - [ - { - finish_reason: "tool_calls", - index: 0, - message: { - content: null, - role: "assistant", - tool_calls: [...toolCallsById.values()], - }, - }, - ], - step.reasoning, - ); - } - - private async handleActionResult( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - if (isToolResult(event.data.result)) { - await this.handleToolResult(event, ctx, event.data.result, hookMetadata); - return; - } - if (isSubagentResult(event.data.result)) { - await this.handleSubagentResult( - event, - ctx, - event.data.result, - hookMetadata, - ); - } - } - - private async handleToolResult( - event: Extract, - ctx: unknown, - result: EveRuntimeToolResultActionResult, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, result.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const tool = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticTool(event, ctx, result, hookMetadata)); - if (!tool) { - return; - } - const failed = - event.data.status === "failed" || - result.isError === true || - event.data.error !== undefined; - tool.span.log({ - ...(failed - ? { - error: actionResultError(event.data.error, result.output), - } - : {}), - metadata: tool.metadata, - output: result.output, + currentLogger()?.updateSpan({ + id: turn.rowId, + metrics: totals, + root_span_id: turn.rootSpanId, + span_id: turn.spanId, }); - - const endTime = eventTime(event); - tool.span.end(endTime === undefined ? undefined : { endTime }); - this.toolsByCallKey.delete(key); - this.completedToolKeys.set(key, true); } - private async handleSubagentCalled( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - if (event.data.remote?.url) { - return; - } - - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; - } - - const key = toolKey(sessionId, event.data.callId); - const metadata = toolMetadataFromTurn(turn); - const existing = this.toolsByCallKey.get(key); - if (existing) { - existing.metadata = { ...existing.metadata, ...metadata }; - existing.span.log({ metadata: existing.metadata }); - return; - } - if (this.completedToolKeys.has(key)) { - return; - } - - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - event.data.callId, - ); - const pending = this.toolsByCallKey.get(key); - if (pending || this.completedToolKeys.has(key)) { - if (pending) { - pending.metadata = { ...pending.metadata, ...metadata }; - pending.span.log({ metadata: pending.metadata }); - } - return; - } - const span = await this.startEveChildSpan(turn.span, { + private async resumeModelSpan( + event: + | EveInstrumentationModelCallCompletedEvent + | EveInstrumentationModelCallFailedEvent, + context: EveInstrumentationHandlerContext, + turn: TurnState, + ): Promise { + const ids = await generateEveIds("step", event.idempotencyKey); + return await this.startEveSpan(context, { event: { - id: eventId, - metadata, + id: ids.rowId, + metadata: this.metadata(event.scope.sessionId), }, - name: event.data.toolName ?? event.data.name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), + name: "eve.step", + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.LLM }, + spanId: ids.spanId, }); } - private async handleSubagentCompleted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, event.data.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const subagent = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticSubagent(event, ctx, hookMetadata)); - if (!subagent) { - return; - } - subagent.span.log({ - ...(event.data.status === "failed" - ? { - error: actionResultError(event.data.error, event.data.output), - } - : {}), - metadata: subagent.metadata, - ...(event.data.output !== undefined ? { output: event.data.output } : {}), - }); - const endTime = eventTime(event); - const recordedEndTime = subagent.span.end( - endTime === undefined ? undefined : { endTime }, + private async resumeActionSpan( + event: + | EveInstrumentationActionCompletedEvent + | EveInstrumentationActionFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const separator = event.idempotencyKey.lastIndexOf(":"); + const callId = + separator < 0 + ? event.idempotencyKey + : event.idempotencyKey.slice(separator + 1); + const ids = await generateEveIds( + "action", + event.scope.sessionId, + event.scope.turnId, + callId, ); - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - spanReferences: normalized.spanReferences.map((reference) => - reference.spanId === subagent.span.spanId - ? { ...reference, endTime: recordedEndTime } - : reference, - ), - }; - }); - } - - private async handleSubagentResult( - event: Extract, - ctx: unknown, - result: Extract, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, result.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const subagent = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticSubagentResult( - event, - ctx, - result, - hookMetadata, - )); - if (!subagent) { - return; - } - const isError = - event.data.status === "failed" || - result.isError === true || - event.data.error !== undefined; - subagent.span.log({ - ...(isError - ? { - error: actionResultError(event.data.error, result.output), - } - : {}), - metadata: subagent.metadata, - output: result.output, + return await this.startEveSpan(context, { + event: { id: ids.rowId, metadata: turn.metadata }, + name: "eve.action", + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + spanId: ids.spanId, }); - const endTime = eventTime(event); - subagent.span.end(endTime === undefined ? undefined : { endTime }); - - this.toolsByCallKey.delete(key); - this.completedToolKeys.set(key, true); } - private handleStepCompleted( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (!step) { - return; - } - - const usage = event.data.usage; - const inputTokens = - typeof usage?.inputTokens === "number" && - Number.isFinite(usage.inputTokens) && - usage.inputTokens >= 0 - ? usage.inputTokens - : undefined; - const outputTokens = - typeof usage?.outputTokens === "number" && - Number.isFinite(usage.outputTokens) && - usage.outputTokens >= 0 - ? usage.outputTokens - : undefined; - const cacheReadTokens = - typeof usage?.cacheReadTokens === "number" && - Number.isFinite(usage.cacheReadTokens) && - usage.cacheReadTokens >= 0 - ? usage.cacheReadTokens - : undefined; - const cacheWriteTokens = - typeof usage?.cacheWriteTokens === "number" && - Number.isFinite(usage.cacheWriteTokens) && - usage.cacheWriteTokens >= 0 - ? usage.cacheWriteTokens - : undefined; - const costUsd = - typeof usage?.costUsd === "number" && - Number.isFinite(usage.costUsd) && - usage.costUsd >= 0 - ? usage.costUsd - : undefined; - const total = - inputTokens !== undefined && outputTokens !== undefined - ? inputTokens + outputTokens - : undefined; - const metrics = { - ...(inputTokens !== undefined ? { prompt_tokens: inputTokens } : {}), - ...(outputTokens !== undefined - ? { completion_tokens: outputTokens } - : {}), - ...(total !== undefined ? { tokens: total } : {}), - ...(cacheReadTokens !== undefined - ? { prompt_cached_tokens: cacheReadTokens } - : {}), - ...(cacheWriteTokens !== undefined - ? { prompt_cache_creation_tokens: cacheWriteTokens } - : {}), - ...(costUsd !== undefined ? { estimated_cost: costUsd } : {}), + private metadata(sessionId: string): Record { + return { + ...(this.configuredMetadata ?? {}), + "eve.session_id": sessionId, }; - step.metrics = { ...step.metrics, ...metrics }; - const sessionId = sessionIdFromContext(ctx); - if (Array.isArray(step.output) && isObject(step.output[0])) { - const finishReason = step.output[0].finish_reason; - if (typeof finishReason !== "string") { - step.output[0].finish_reason = normalizedFinishReason( - event.data.finishReason, - ); - } - } - step.span.log({ - ...(step.input !== undefined ? { input: step.input } : {}), - metadata: step.metadata, - metrics, - output: step.output, - }); - const endTime = eventTime(event); - step.span.end(endTime === undefined ? undefined : { endTime }); - - const turn = this.turnForEvent(event, ctx); - if (turn) { - for (const [key, value] of Object.entries(metrics)) { - turn.metrics[key] = (turn.metrics[key] ?? 0) + value; - } - turn.stepsByIndex.delete(event.data.stepIndex); - } - this.markStepEnded(event.data.turnId, event.data.stepIndex); - if (sessionId) { - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - } - } - - private handleStepFailed( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (step) { - step.span.log({ - error: errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ), - }); - const endTime = eventTime(event); - step.span.end(endTime === undefined ? undefined : { endTime }); - } - - const turn = this.turnForEvent(event, ctx); - turn?.stepsByIndex.delete(event.data.stepIndex); - this.markStepEnded(event.data.turnId, event.data.stepIndex); - const sessionId = sessionIdFromContext(ctx); - if (sessionId) { - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - } } +} - private handleTurnCompleted( - event: Extract, - ctx: unknown, - ): void { - const turn = this.turnForEvent(event, ctx); - if (!turn) { - return; - } - - this.finalizeTurn(turn, { - endTime: eventTime(event), - }); +function readSpanReference(value: unknown): EveSpanReference | undefined { + if (!isObject(value)) { + return undefined; } - - private handleTurnFailed( - event: Extract, - ctx: unknown, - ): void { - const turn = this.turnForEvent(event, ctx); - if (!turn) { - return; - } - - this.finalizeTurn(turn, { - endTime: eventTime(event), - error: errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ), - }); + const exported = value["exported"]; + const rootSpanId = value["rootSpanId"]; + const spanId = value["spanId"]; + if ( + typeof exported !== "string" || + typeof rootSpanId !== "string" || + typeof spanId !== "string" + ) { + return undefined; } - - private handleSessionFailed( - event: Extract, - ctx: unknown, - ): void { - const sessionId = event.data.sessionId || sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - const error = errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ); - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - this.finalizeTurn(turn, { - endTime: eventTime(event), - error, - }); - } - - for (const [key, tool] of this.toolsByCallKey) { - if (key.startsWith(`${sessionId}:`)) { - const endTime = eventTime(event); - if (!tool.endedByTurn) { - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; + return { + exported, + rootSpanId, + spanId, + ...(isObject(value["turnMetricContributions"]) + ? { + turnMetricContributions: numericMetricContributions( + value["turnMetricContributions"], + ), } - } - } - } - - private handleSessionCompleted( - event: Extract, - ctx: unknown, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - this.finalizeTurn(turn, { - endTime: eventTime(event), - }); - } + : {}), + }; +} - for (const [key, tool] of this.toolsByCallKey) { - if (key.startsWith(`${sessionId}:`) && !tool.endedByTurn) { - const endTime = eventTime(event); - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; - } - } +function capturedModelInput( + input: EveInstrumentationModelInput | undefined, +): unknown[] | undefined { + if (input === undefined) { + return undefined; } - - private async ensureTurn( - event: Extract< - EveHandleMessageStreamEvent, - { - data: { readonly sequence: number; readonly turnId: string }; + const value: unknown[] = []; + if (typeof input.instructions === "string") { + value.push({ content: input.instructions, role: "system" }); + } else if (Array.isArray(input.instructions)) { + for (const instruction of input.instructions) { + const projected = projectedModelMessage(instruction); + if (projected?.role === "system") { + value.push(projected); } - >, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return undefined; } - - const key = turnKey(sessionId, event.data.turnId); - const existing = this.turnsByKey.get(key); - if (existing) { - return existing; + } else if (isObject(input.instructions)) { + const instruction = projectedModelMessage(input.instructions); + if (instruction?.role === "system") { + value.push(instruction); } - - const metadata = { - ...readEveTraceState(this.state).metadata, - ...(hookMetadata ?? {}), - "eve.session_id": sessionId, - }; - const span = await this.startTurnSpan(sessionId, event, ctx, metadata); - span.log({ metadata }); - const state = { - key, - metadata, - metrics: {}, - sessionId, - span, - stepsByIndex: new Map(), - turnId: event.data.turnId, - }; - this.turnsByKey.set(key, state); - return state; } - - private async startRequestedTool( - event: Extract, - turn: TurnState, - sessionId: string, - action: EveRuntimeToolCallActionRequest, - ): Promise { - const key = toolKey(sessionId, action.callId); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; + for (const message of input.messages) { + const projected = projectedModelMessage(message); + if (projected !== undefined) { + value.push(projected); } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "tool", - sessionId, - event.data.turnId, - action.callId, - ); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - input: action.input, - metadata, - }, - name: action.toolName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ input: action.input, metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }); } + return value; +} - private async startRequestedSubagent( - event: Extract, - turn: TurnState, - sessionId: string, - action: Extract, - ): Promise { - const key = toolKey(sessionId, action.callId); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - - const name = action.subagentName ?? action.name ?? "agent"; - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - action.callId, - ); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - input: action.input, - metadata, - }, - name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ input: action.input, metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }); - } - - private async startSyntheticTool( - event: Extract, - ctx: unknown, - result: EveRuntimeToolResultActionResult, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "tool", - sessionId, - event.data.turnId, - result.callId, - ); - const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId)); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: result.toolName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, result.callId), state); - return state; - } - - private async startSyntheticSubagent( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - event.data.callId, - ); - const existing = this.toolsByCallKey.get( - toolKey(sessionId, event.data.callId), - ); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: event.data.subagentName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, event.data.callId), state); - return state; - } - - private async startSyntheticSubagentResult( - event: Extract, - ctx: unknown, - result: Extract, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - result.callId, - ); - const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId)); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: result.subagentName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, result.callId), state); - return state; +function projectedModelMessage( + value: unknown, +): { content: unknown; role: string } | undefined { + if (!isObject(value)) { + return undefined; } - - private async startTurnSpan( - sessionId: string, - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly sequence: number; readonly turnId: string } } - >, - ctx: unknown, - metadata: Record, - ): Promise { - const session = isObject(ctx) ? ctx["session"] : undefined; - const parent = isObject(session) ? session["parent"] : undefined; - const parentTurn = isObject(parent) ? parent["turn"] : undefined; - const parentLineage = - isObject(parent) && - typeof parent["callId"] === "string" && - typeof parent["sessionId"] === "string" && - isObject(parentTurn) && - typeof parentTurn["id"] === "string" - ? { - callId: parent["callId"], - sessionId: parent["sessionId"], - turnId: parentTurn["id"], - } - : undefined; - const [{ rowId: eventId, spanId }, rootSpanId, parentSpanId] = - await Promise.all([ - generateEveIds("turn", sessionId, event.data.turnId), - deterministicEveId( - "eve:root", - parentLineage?.sessionId ?? sessionId, - parentLineage?.turnId ?? event.data.turnId, - ), - parentLineage - ? deterministicEveId( - "eve:subagent", - parentLineage.sessionId, - parentLineage.callId, - ) - : Promise.resolve(undefined), - ]); - - return await this.startEveSpan({ - event: { - id: eventId, - metadata, - }, - name: "eve.turn", - parentSpanIds: parentSpanId - ? { rootSpanId, spanId: parentSpanId } - : { parentSpanIds: [], rootSpanId }, - spanAttributes: { type: SpanTypeAttribute.TASK }, - spanId, - startTime: eventTime(event), - }); + const role = value["role"]; + if ( + role !== "system" && + role !== "user" && + role !== "assistant" && + role !== "tool" + ) { + return undefined; } - - private turnForEvent( - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly turnId: string } } - >, - ctx: unknown, - ): TurnState | undefined { - const sessionId = sessionIdFromContext(ctx); - return sessionId - ? this.turnsByKey.get(turnKey(sessionId, event.data.turnId)) - : undefined; + const content = value["content"]; + if (typeof content === "string") { + return { content, role }; } - - private stepForEvent( - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly stepIndex: number; readonly turnId: string } } - >, - ctx: unknown, - ): StepState | undefined { - return this.turnForEvent(event, ctx)?.stepsByIndex.get( - event.data.stepIndex, - ); + if (!Array.isArray(content)) { + return undefined; } - - private finalizeTurn( - turn: TurnState, - args: { endTime: number | undefined; error?: Error }, - ): void { - const { endTime } = args; - for (const step of turn.stepsByIndex.values()) { - step.span.log({ - ...(step.input !== undefined ? { input: step.input } : {}), - metadata: step.metadata, - metrics: step.metrics, - output: step.output, - }); - step.span.end(endTime === undefined ? undefined : { endTime }); - } - turn.stepsByIndex.clear(); - - for (const tool of this.toolsByCallKey.values()) { - if (tool.turnKey !== turn.key) { - continue; + const projected: unknown[] = []; + for (const part of content) { + if (!isObject(part) || typeof part["type"] !== "string") { + continue; + } + switch (part["type"]) { + case "text": + case "reasoning": + if (typeof part["text"] === "string") { + projected.push({ text: part["text"], type: part["type"] }); + } + break; + case "image": { + if (part["image"] === undefined) { + break; + } + const mediaType = + typeof part["mediaType"] === "string" + ? part["mediaType"] + : "image/png"; + projected.push({ + image: mediaAttachment(part["image"], mediaType, undefined), + ...(typeof part["mediaType"] === "string" ? { mediaType } : {}), + type: "image", + }); + break; } - if (tool.endedByTurn) { - continue; + case "file": { + const mediaType = + typeof part["mediaType"] === "string" + ? part["mediaType"] + : "application/octet-stream"; + const data = + part["data"] === undefined + ? undefined + : mediaAttachment( + part["data"], + mediaType, + typeof part["filename"] === "string" + ? part["filename"] + : undefined, + ); + projected.push({ + ...(data === undefined ? {} : { data }), + ...(typeof part["filename"] === "string" + ? { filename: part["filename"] } + : {}), + mediaType, + type: "file", + }); + break; } - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; + case "tool-call": + if ( + typeof part["toolCallId"] === "string" && + typeof part["toolName"] === "string" + ) { + projected.push({ + input: part["input"], + toolCallId: part["toolCallId"], + toolName: part["toolName"], + type: "tool-call", + }); + } + break; + case "tool-result": + if ( + typeof part["toolCallId"] === "string" && + typeof part["toolName"] === "string" + ) { + projected.push({ + output: projectedToolOutput(part["output"]), + toolCallId: part["toolCallId"], + toolName: part["toolName"], + type: "tool-result", + }); + } + break; + case "tool-approval-response": + if ( + typeof part["approvalId"] === "string" && + typeof part["approved"] === "boolean" + ) { + projected.push({ + approvalId: part["approvalId"], + approved: part["approved"], + ...(typeof part["reason"] === "string" + ? { reason: part["reason"] } + : {}), + type: "tool-approval-response", + }); + } + break; + default: + break; } + } + return { content: projected, role }; +} - if (args.error) { - turn.span.log({ error: args.error }); - } else { - turn.span.log({ - metadata: turn.metadata, - metrics: turn.metrics, - output: turn.output, - }); - } - turn.span.end(endTime === undefined ? undefined : { endTime }); - this.turnsByKey.delete(turn.key); - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); +function projectedToolOutput(value: unknown): unknown { + if (!isObject(value)) { + return value; + } + if (typeof value["type"] !== "string") { + return undefined; + } + switch (value["type"]) { + case "text": + case "json": + case "error-text": + case "error-json": + return { type: value["type"], value: value["value"] }; + case "execution-denied": return { - ...normalized, - reasoningBlocks: normalized.reasoningBlocks.filter( - (entry) => - !entry.key.startsWith(`${turn.sessionId}\0${turn.turnId}\0`), - ), - stepStarts: normalized.stepStarts.filter( - (entry) => entry.turnId !== turn.turnId, - ), + ...(typeof value["reason"] === "string" + ? { reason: value["reason"] } + : {}), + type: "execution-denied", }; - }); - } - - private cleanupSession(sessionId: string): void { - const keyPrefix = `${sessionId}:`; - for (const key of this.turnsByKey.keys()) { - if (key.startsWith(keyPrefix)) { - this.turnsByKey.delete(key); - } - } - for (const key of this.toolsByCallKey.keys()) { - if (key.startsWith(keyPrefix)) { - this.toolsByCallKey.delete(key); + case "content": { + if (!Array.isArray(value["value"])) { + return { type: "content", value: [] }; } - } - for (const key of this.completedToolKeys.keys()) { - if (key.startsWith(keyPrefix)) { - this.completedToolKeys.delete(key); + const content: unknown[] = []; + for (const part of value["value"]) { + if (!isObject(part) || typeof part["type"] !== "string") { + continue; + } + if (part["type"] === "text" && typeof part["text"] === "string") { + content.push({ text: part["text"], type: "text" }); + } else if ( + (part["type"] === "file-data" || part["type"] === "image-data") && + part["data"] !== undefined && + typeof part["mediaType"] === "string" + ) { + const data = mediaAttachment( + part["data"], + part["mediaType"], + typeof part["filename"] === "string" ? part["filename"] : undefined, + ); + content.push({ + ...(data === undefined ? {} : { data }), + ...(typeof part["filename"] === "string" + ? { filename: part["filename"] } + : {}), + mediaType: part["mediaType"], + type: part["type"], + }); + } else if ( + (part["type"] === "file-url" || part["type"] === "image-url") && + typeof part["url"] === "string" + ) { + content.push({ type: part["type"], url: part["url"] }); + } } + return { + type: "content", + value: content, + }; } - this.state.update(() => emptyEveTraceState()); - } - - private async flushInstrumentation(): Promise { - try { - await flush(); - return true; - } catch (error) { - debugLogger.warn("Error in Eve flush instrumentation:", error); - return false; - } - } -} - -function emptyEveTraceState(): EveTraceState { - return { - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], - }; -} - -function normalizeEveTraceState(state: unknown): EveTraceState { - if (!isObject(state)) { - return emptyEveTraceState(); + default: + return undefined; } - const metadata = isObject(state["metadata"]) ? state["metadata"] : {}; - const spanReferences = Array.isArray(state["spanReferences"]) - ? state["spanReferences"] - .flatMap((entry): EveTraceState["spanReferences"] => { - if (!isObject(entry)) { - return []; - } - const exported = entry["exported"]; - const endTime = entry["endTime"]; - const rootSpanId = entry["rootSpanId"]; - const rowId = entry["rowId"]; - const spanId = entry["spanId"]; - const startEvent = entry["startEvent"]; - const startEventCreated = isObject(startEvent) - ? startEvent["created"] - : undefined; - const startEventMetrics = isObject(startEvent) - ? startEvent["metrics"] - : undefined; - const startEventSpanAttributes = isObject(startEvent) - ? startEvent["span_attributes"] - : undefined; - const startEventSpanParents = isObject(startEvent) - ? startEvent["span_parents"] - : undefined; - const normalizedStartEvent = - typeof startEventCreated === "string" && - isObject(startEventMetrics) && - typeof startEventMetrics["start"] === "number" && - Number.isFinite(startEventMetrics["start"]) && - isObject(startEventSpanAttributes) && - Array.isArray(startEventSpanParents) && - startEventSpanParents.every( - (parent): parent is string => typeof parent === "string", - ) - ? { - created: startEventCreated, - metrics: { start: startEventMetrics["start"] }, - span_attributes: { ...startEventSpanAttributes }, - span_parents: [...startEventSpanParents], - } - : undefined; - return typeof exported === "string" && - typeof rootSpanId === "string" && - typeof rowId === "string" && - typeof spanId === "string" - ? [ - { - ...(typeof endTime === "number" && Number.isFinite(endTime) - ? { endTime } - : {}), - exported, - rootSpanId, - rowId, - spanId, - ...(normalizedStartEvent - ? { startEvent: normalizedStartEvent } - : {}), - }, - ] - : []; - }) - .slice(-MAX_STORED_SPAN_REFERENCES) - : []; - const llmInputs = Array.isArray(state["llmInputs"]) - ? state["llmInputs"] - .flatMap((entry): EveTraceState["llmInputs"] => { - if (!isObject(entry)) { - return []; - } - const key = entry["key"]; - const input = entry["input"]; - return typeof key === "string" && isCapturedModelInput(input) - ? [{ input, key }] - : []; - }) - .slice(-MAX_STORED_LLM_INPUTS) - : []; - const reasoningBlocks = Array.isArray(state["reasoningBlocks"]) - ? state["reasoningBlocks"] - .flatMap((entry): EveTraceState["reasoningBlocks"] => { - if (!isObject(entry)) { - return []; - } - const content = entry["content"]; - const eventAt = entry["eventAt"]; - const key = entry["key"]; - return typeof content === "string" && - (eventAt === undefined || typeof eventAt === "string") && - typeof key === "string" - ? [ - { - content, - ...(typeof eventAt === "string" ? { eventAt } : {}), - key, - }, - ] - : []; - }) - .slice(-MAX_STORED_REASONING_BLOCKS) - : []; - const stepStarts = Array.isArray(state["stepStarts"]) - ? state["stepStarts"] - .flatMap((entry): EveTraceState["stepStarts"] => { - if (!isObject(entry)) { - return []; - } - const ordinal = entry["ordinal"]; - const open = entry["open"]; - const stepIndex = entry["stepIndex"]; - const turnId = entry["turnId"]; - return typeof ordinal === "number" && - Number.isInteger(ordinal) && - ordinal >= 0 && - typeof open === "boolean" && - typeof stepIndex === "number" && - Number.isInteger(stepIndex) && - typeof turnId === "string" - ? [{ open, ordinal, stepIndex, turnId }] - : []; - }) - .slice(-MAX_STORED_STEP_STARTS) - : []; - return { - llmInputs, - metadata: { ...metadata }, - reasoningBlocks, - spanReferences, - stepStarts, - }; } -function readEveTraceState( - state: EveStateHandle, -): EveTraceState { - try { - return normalizeEveTraceState(state.get()); - } catch { - return emptyEveTraceState(); +function mediaAttachment( + data: unknown, + contentType: string, + filename: string | undefined, +): unknown { + const blob = convertDataToBlob(data, contentType); + if (blob === null) { + return data; } -} - -function storeEveReasoning( - state: EveStateHandle, - sessionId: string, - event: Extract, -): readonly EveReasoningBlock[] { - const eventAt = event.meta?.at; - const key = llmInputKey(sessionId, event.data.turnId, event.data.stepIndex); - let stored: readonly EveReasoningBlock[] = []; - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const alreadyStored = normalized.reasoningBlocks.some( - (entry) => - entry.content === event.data.reasoning && - entry.eventAt === eventAt && - entry.key === key, - ); - const reasoningBlocks = alreadyStored - ? normalized.reasoningBlocks - : [ - ...normalized.reasoningBlocks, - { - content: event.data.reasoning, - ...(eventAt ? { eventAt } : {}), - key, - }, - ].slice(-MAX_STORED_REASONING_BLOCKS); - stored = reasoningBlocks.flatMap((entry) => - entry.key === key - ? [ - { - content: entry.content, - ...(entry.eventAt ? { eventAt: entry.eventAt } : {}), - }, - ] - : [], - ); - return alreadyStored ? normalized : { ...normalized, reasoningBlocks }; + return new Attachment({ + contentType, + data: blob, + filename: + filename ?? `attachment.${getExtensionFromMediaType(contentType)}`, }); - return stored; } -function readStoredEveReasoning( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): readonly EveReasoningBlock[] { - const key = llmInputKey(sessionId, turnId, stepIndex); - return readEveTraceState(state).reasoningBlocks.flatMap((entry) => - entry.key === key - ? [ - { - content: entry.content, - ...(entry.eventAt ? { eventAt: entry.eventAt } : {}), - }, - ] - : [], - ); +function initialTurnInput(input: readonly unknown[]): unknown[] | undefined { + for (let index = input.length - 1; index >= 0; index--) { + const message = input[index]; + if (isObject(message) && message["role"] === "user") { + return [message]; + } + } + return undefined; } -function clearStoredEveReasoning( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): void { - const key = llmInputKey(sessionId, turnId, stepIndex); - state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - reasoningBlocks: normalized.reasoningBlocks.filter( - (entry) => entry.key !== key, - ), - }; +function modelOutput( + finishReason: string, + content: readonly EveInstrumentationContentPart[] | undefined, +): unknown { + const parts = content ?? []; + const text = parts + .filter( + ( + part, + ): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + const reasoning = parts + .filter( + ( + part, + ): part is Extract< + EveInstrumentationContentPart, + { type: "reasoning" } + > => part.type === "reasoning", + ) + .map((part) => ({ content: part.text })); + const toolCalls = parts.flatMap((part) => { + if (part.type !== "tool-call") { + return []; + } + return [ + { + function: { + arguments: jsonString(part.input), + name: part.toolName, + }, + id: part.callId, + type: "function", + }, + ]; }); -} - -function eveOutputMessage(output: unknown): unknown { - return Array.isArray(output) && isObject(output[0]) - ? output[0]["message"] - : undefined; -} - -function mergeEveReasoning( - output: unknown, - reasoning: readonly { content: string }[], -): unknown | undefined { - if (reasoning.length === 0) { - return output; - } - - const choice = Array.isArray(output) && isObject(output[0]) ? output[0] : {}; - const message = isObject(choice["message"]) ? choice["message"] : {}; return [ { - ...choice, - index: typeof choice["index"] === "number" ? choice["index"] : 0, + finish_reason: normalizedFinishReason(finishReason), + index: 0, message: { - ...message, - content: "content" in message ? message["content"] : null, - reasoning: reasoning.map((block) => ({ content: block.content })), - role: - typeof message["role"] === "string" ? message["role"] : "assistant", + content: text.length === 0 ? null : text, + ...(reasoning.length === 0 ? {} : { reasoning }), + role: "assistant", + ...(toolCalls.length === 0 ? {} : { tool_calls: toolCalls }), }, }, ]; } -function captureEveModelInput( - state: EveStateHandle, - input: EveInstrumentationStepStartedEventInput, -): void { - const sessionId = input.session.id; - const turnId = input.turn.id; - const stepIndex = input.step.index; - const captured = capturedModelInput(input.modelInput); - if (!captured) { - return; - } - - const key = llmInputKey(sessionId, turnId, stepIndex); - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const llmInputs = [...normalized.llmInputs, { input: captured, key }]; - return { - ...normalized, - llmInputs: llmInputs.slice(-MAX_STORED_LLM_INPUTS), - }; - }); -} - -function consumeCapturedEveModelInput( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): CapturedEveModelInput | undefined { - try { - const key = llmInputKey(sessionId, turnId, stepIndex); - let input: CapturedEveModelInput | undefined; - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const index = normalized.llmInputs.findIndex( - (candidate) => candidate.key === key, - ); - if (index < 0) { - return normalized; - } - input = normalized.llmInputs[index]?.input; - return { - ...normalized, - llmInputs: normalized.llmInputs.filter( - (_, candidateIndex) => candidateIndex !== index, - ), - }; - }); - return input; - } catch (error) { - debugLogger.warn("Error in Eve LLM input consumption:", error); - return undefined; - } +function finalText( + content: readonly EveInstrumentationContentPart[] | undefined, +): string | undefined { + const text = (content ?? []) + .filter( + ( + part, + ): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + return text.length === 0 ? undefined : text; } -function capturedModelInput( - modelInput: EveInstrumentationModelInput, -): CapturedEveModelInput | undefined { - const { instructions, messages } = modelInput; - const value: CapturedEveModelMessage[] = []; - if (typeof instructions === "string") { - value.push({ content: instructions, role: "system" }); - } else if (instructions) { - value.push(...instructions.map(capturedEveModelMessage)); - } - value.push(...messages.map(capturedEveModelMessage)); - +function jsonString(value: unknown): string { try { - const cloned: unknown = JSON.parse(JSON.stringify(value)); - if (!Array.isArray(cloned)) { - return undefined; - } - return cloned; + return JSON.stringify(value) ?? "null"; } catch { - return undefined; + return "null"; } } -function capturedEveModelMessage( - message: EveModelMessage, -): CapturedEveModelMessage { - const { content, role } = message; - if (typeof content === "string") { - return { content, role }; +function normalizedFinishReason(finishReason: string): string { + if (finishReason === "content-filter") { + return "content_filter"; } - return { content: content.map(capturedEveModelContentPart), role }; -} - -function capturedEveModelContentPart( - part: EveModelMessageContentPart, -): Record { - switch (part.type) { - case "text": - case "reasoning": - return { text: part.text, type: part.type }; - case "image": - return { - image: part.image, - ...(part.mediaType !== undefined ? { mediaType: part.mediaType } : {}), - type: "image", - }; - case "file": - case "reasoning-file": - return { - data: part.data, - ...(part.type === "file" && part.filename !== undefined - ? { filename: part.filename } - : {}), - mediaType: part.mediaType, - type: part.type, - }; - case "custom": - return { - ...("kind" in part ? { kind: part.kind } : {}), - type: "custom", - }; - case "tool-call": - return { - input: part.input, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } - : {}), - toolCallId: part.toolCallId, - toolName: part.toolName, - type: "tool-call", - }; - case "tool-result": { - const output = part.output; - let capturedOutput: Record; - switch (output.type) { - case "text": - case "error-text": - capturedOutput = { type: output.type, value: output.value }; - break; - case "json": - case "error-json": - capturedOutput = { type: output.type, value: output.value }; - break; - case "execution-denied": - capturedOutput = { - ...(output.reason !== undefined ? { reason: output.reason } : {}), - type: "execution-denied", - }; - break; - case "content": - capturedOutput = { - type: "content", - value: output.value.map(capturedEveModelContentPart), - }; - break; - } - return { - output: capturedOutput, - toolCallId: part.toolCallId, - toolName: part.toolName, - type: "tool-result", - }; - } - case "tool-approval-request": - return { - approvalId: part.approvalId, - ...(part.isAutomatic !== undefined - ? { isAutomatic: part.isAutomatic } - : {}), - ...(part.signature !== undefined ? { signature: part.signature } : {}), - toolCallId: part.toolCallId, - type: "tool-approval-request", - }; - case "tool-approval-response": - return { - approvalId: part.approvalId, - approved: part.approved, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } - : {}), - ...(part.reason !== undefined ? { reason: part.reason } : {}), - type: "tool-approval-response", - }; - case "file-data": - case "image-data": - return { - data: part.data, - ...(part.type === "file-data" && part.filename !== undefined - ? { filename: part.filename } - : {}), - mediaType: part.mediaType, - type: part.type, - }; - case "file-url": - case "image-url": - return { - ...(part.type === "file-url" && part.mediaType !== undefined - ? { mediaType: part.mediaType } - : {}), - type: part.type, - url: part.url, - }; - case "file-id": - case "image-file-id": - return { fileId: part.fileId, type: part.type }; - case "file-reference": - case "image-file-reference": - return { - providerReference: part.providerReference, - type: part.type, - }; + if (finishReason === "tool-calls") { + return "tool_calls"; } + return finishReason; } -function isCapturedModelInput(input: unknown): input is CapturedEveModelInput { - return ( - Array.isArray(input) && - input.every( - (message) => - isObject(message) && - (message["role"] === "system" || - message["role"] === "user" || - message["role"] === "assistant" || - message["role"] === "tool") && - (typeof message["content"] === "string" || - (Array.isArray(message["content"]) && - message["content"].every(isObject))), - ) +function usageMetrics(usage: EveInstrumentationUsage): Record { + const inputTokens = validMetric(usage.inputTokens); + const outputTokens = validMetric(usage.outputTokens); + const cacheReadTokens = validMetric(usage.inputTokenDetails?.cacheReadTokens); + const cacheWriteTokens = validMetric( + usage.inputTokenDetails?.cacheWriteTokens, ); + return { + ...(inputTokens === undefined ? {} : { prompt_tokens: inputTokens }), + ...(outputTokens === undefined ? {} : { completion_tokens: outputTokens }), + ...(inputTokens === undefined || outputTokens === undefined + ? {} + : { tokens: inputTokens + outputTokens }), + ...(cacheReadTokens === undefined + ? {} + : { prompt_cached_tokens: cacheReadTokens }), + ...(cacheWriteTokens === undefined + ? {} + : { prompt_cache_creation_tokens: cacheWriteTokens }), + }; } -function llmInputKey( - sessionId: string, - turnId: string, - stepIndex: number, -): string { - return `${sessionId}\0${turnId}\0${stepIndex}`; +function validMetric(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : undefined; } -function modelMetadataFromRuntime(runtime: unknown): Record { - if (!isObject(runtime)) { - return {}; +function numericMetrics( + value: Record, +): Record { + const metrics: Record = {}; + for (const name of [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "tokens", + ]) { + const metric = validMetric(value[name]); + if (metric !== undefined) { + metrics[name] = metric; + } } - const modelId = runtime["modelId"]; - return typeof modelId === "string" && !modelId.trim().startsWith("dynamic:") - ? modelMetadataFromModelId(modelId) - : {}; + return metrics; } -function modelMetadataFromModelId(modelId: string): Record { - const normalized = modelId.trim(); - if (!normalized) { - return {}; - } - - const slashIndex = normalized.indexOf("/"); - if (slashIndex > 0 && slashIndex < normalized.length - 1) { - return { - model: normalized.slice(slashIndex + 1), - provider: normalized.slice(0, slashIndex), - }; +function numericMetricContributions( + value: Record, +): Record> { + const contributions: Record> = {}; + for (const [idempotencyKey, contribution] of Object.entries(value)) { + if (isObject(contribution)) { + contributions[idempotencyKey] = numericMetrics(contribution); + } } + return contributions; +} +function modelMetadata( + modelId: string, + providerId: string, +): Record { + const provider = providerId.split(".")[0]?.trim(); + const model = modelId.trim(); return { - model: normalized, + ...(model.length === 0 ? {} : { model }), + ...(provider === undefined || provider.length === 0 ? {} : { provider }), }; } -function sessionIdFromContext(ctx: unknown): string | undefined { - if (!isObject(ctx)) { - return undefined; +function toError(error: unknown, fallback: string): Error { + if (error instanceof Error) { + return error; } - const session = ctx["session"]; - if (!isObject(session)) { - return undefined; + if (typeof error === "string" && error.length > 0) { + return new Error(error); } - const id = session["id"]; - return typeof id === "string" ? id : undefined; -} - -function toolMetadataFromTurn(turn: TurnState): Record { - const { model: _model, provider: _provider, ...metadata } = turn.metadata; - return metadata; -} - -function isToolCallAction( - action: unknown, -): action is EveRuntimeToolCallActionRequest { - return ( - isObject(action) && - action["kind"] === "tool-call" && - typeof action["callId"] === "string" && - typeof action["toolName"] === "string" && - isObject(action["input"]) - ); -} - -function isLocalSubagentCallAction( - action: unknown, -): action is Extract { - return ( - isObject(action) && - action["kind"] === "subagent-call" && - typeof action["callId"] === "string" && - isObject(action["input"]) - ); -} - -function isTraceableActionRequest( - action: unknown, -): action is - | EveRuntimeToolCallActionRequest - | Extract { - return isToolCallAction(action) || isLocalSubagentCallAction(action); -} - -function isToolResult( - result: unknown, -): result is EveRuntimeToolResultActionResult { - return ( - isObject(result) && - result["kind"] === "tool-result" && - typeof result["callId"] === "string" && - typeof result["toolName"] === "string" - ); -} - -function isSubagentResult( - result: unknown, -): result is Extract { - return ( - isObject(result) && - result["kind"] === "subagent-result" && - typeof result["callId"] === "string" && - typeof result["subagentName"] === "string" - ); -} - -function normalizedFinishReason( - finishReason: EveAssistantStepFinishReason, -): string { - switch (finishReason) { - case "content-filter": - return "content_filter"; - case "tool-calls": - return "tool_calls"; - default: - return finishReason; - } -} - -function errorFromMessage( - message: string, - code: string, - details?: unknown, -): Error { - const error = new Error(`${code}: ${message}`); - if (details !== undefined) { - error.cause = details; - } - return error; -} - -function actionResultError( - error: EveActionResultError | undefined, - output: unknown, -): Error { - if (error) { - return errorFromMessage(error.message, error.code); + if (isObject(error) && typeof error["message"] === "string") { + const code = typeof error["code"] === "string" ? error["code"] + ": " : ""; + const result = new Error(code + error["message"]); + result.cause = error; + return result; } - const result = new Error("Eve action failed"); - if (output !== undefined) { - result.cause = output; + const result = new Error(fallback); + if (error !== undefined) { + result.cause = error; } return result; } -function eventTime(event: { - readonly meta?: { readonly at: string }; -}): number | undefined { - if (!event.meta?.at) { - return undefined; - } - const timestamp = Date.parse(event.meta.at); - return Number.isFinite(timestamp) ? timestamp / 1000 : undefined; +function actionEndArgs( + acceptedAtMs: number | undefined, +): { endTime: number } | undefined { + return typeof acceptedAtMs === "number" && Number.isFinite(acceptedAtMs) + ? { endTime: acceptedAtMs / 1000 } + : undefined; } function turnKey(sessionId: string, turnId: string): string { - return `${sessionId}:${turnId}`; -} - -function toolKey(sessionId: string, callId: string): string { - return `${sessionId}:${callId}`; + return sessionId + ":" + turnId; } async function generateEveIds( - kind: EveEntityKind, + kind: "action" | "step" | "turn", ...parts: string[] ): Promise<{ rowId: string; spanId: string }> { const [rowId, spanId] = await Promise.all([ - deterministicEveId(`eve:row:${kind}`, ...parts), - deterministicEveId(`eve:${kind}`, ...parts), + deterministicEveId("eve:row:" + kind, ...parts), + deterministicEveId("eve:" + kind, ...parts), ]); return { rowId, spanId }; } async function deterministicEveId(...parts: string[]): Promise { const data = new TextEncoder().encode( - parts.map((part) => `${part.length}:${part}`).join("\0"), + parts.map((part) => part.length + ":" + part).join("\0"), ); const digest = await globalThis.crypto.subtle.digest("SHA-256", data); const bytes = Array.from(new Uint8Array(digest, 0, 16)); const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0")).join(""); - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + return ( + hex.slice(0, 8) + + "-" + + hex.slice(8, 12) + + "-" + + hex.slice(12, 16) + + "-" + + hex.slice(16, 20) + + "-" + + hex.slice(20) + ); } diff --git a/js/src/vendor-sdk-types/eve.ts b/js/src/vendor-sdk-types/eve.ts index 1910b3a87..5499fceb9 100644 --- a/js/src/vendor-sdk-types/eve.ts +++ b/js/src/vendor-sdk-types/eve.ts @@ -1,8 +1,8 @@ /** - * Vendored types for eve's authored hook APIs. + * Vendored types for eve's instrumentation provider API, introduced in 0.34.0. * * Keep this surface intentionally narrow. These types are not exported to SDK - * users and should only cover fields we read, correlate, or log. + * users and should only cover fields the Braintrust provider reads or returns. */ export type EveJsonValue = @@ -13,499 +13,197 @@ export type EveJsonValue = | EveJsonValue[] | { readonly [key: string]: EveJsonValue }; -export type EveJsonObject = { readonly [key: string]: EveJsonValue }; - -export interface EveHookContext { - readonly session: { - readonly id: string; - readonly parent?: { - readonly callId?: string; - readonly sessionId?: string; - readonly turn?: { - readonly id?: string; - }; - }; - }; +export interface EveInstrumentationState { + get(): EveJsonValue | undefined; + set(value: EveJsonValue | undefined): void; } -export type EveAssistantStepFinishReason = - | "content-filter" - | "error" - | "length" - | "other" - | "stop" - | "tool-calls"; +export interface EveInstrumentationHandlerContext { + readonly state: EveInstrumentationState; +} -export interface EveStreamEventMeta { - readonly at: string; +export interface EveInstrumentationSetupContext { + readonly agentName: string; + readonly environment?: "development" | "preview" | "production"; + readonly evaluation?: { readonly runId: string }; + readonly frameworkVersion?: string; } -export interface EveRuntimeToolCallActionRequest { - readonly callId: string; - readonly input: EveJsonObject; - readonly kind: "tool-call"; - readonly toolName: string; +export interface EveInstrumentationAttemptScope { + readonly attemptId: string; + readonly attemptIndex: number; + readonly rootSessionId?: string; + readonly sessionId: string; + readonly stepIndex: number; + readonly turnId: string; } -export interface EveRuntimeToolResultActionResult { +export interface EveInstrumentationParentLineage { readonly callId: string; - readonly isError?: boolean; - readonly kind: "tool-result"; - readonly output: EveJsonValue; - readonly toolName: string; + readonly sessionId: string; + readonly turnId: string; } -export type EveRuntimeActionRequest = - | EveRuntimeToolCallActionRequest - | { - readonly callId: string; - readonly input?: EveJsonObject; - readonly kind: "load-skill" | "remote-agent-call"; - readonly name?: string; - } - | { - readonly callId: string; - readonly input: EveJsonObject; - readonly kind: "subagent-call"; - readonly name?: string; - readonly subagentName?: string; - }; - -export type EveRuntimeActionResult = - | EveRuntimeToolResultActionResult - | { - readonly callId: string; - readonly isError?: boolean; - readonly kind: "load-skill-result"; - readonly output?: EveJsonValue; - readonly name?: string; - } - | { - readonly callId: string; - readonly isError?: boolean; - readonly kind: "subagent-result"; - readonly output?: EveJsonValue; - readonly subagentName?: string; - }; +export interface EveInstrumentationTraceContext { + readonly spanId: string; + readonly traceFlags: number; + readonly traceId: string; +} -export type EveActionResultStatus = "completed" | "failed" | "rejected"; +export interface EveInstrumentationUsage { + readonly inputTokenDetails?: { + readonly cacheReadTokens?: number; + readonly cacheWriteTokens?: number; + }; + readonly inputTokens?: number; + readonly outputTokens?: number; +} -export interface EveActionResultError { - readonly code: string; - readonly message: string; +export interface EveInstrumentationModelInput { + readonly instructions?: unknown; + readonly messages: readonly unknown[]; } -export type EveHandleMessageStreamEvent = - | { - readonly data: { - readonly invocation?: unknown; - readonly runtime?: { - readonly agentId: string; - readonly agentName?: string; - readonly eveVersion: string; - readonly modelId: string; - }; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.started"; - } +export type EveInstrumentationContentPart = | { - readonly data: { - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.started"; + readonly text: string; + readonly type: "text"; } | { - readonly data: { - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.completed"; + readonly text: string; + readonly type: "reasoning"; } | { - readonly data: { - readonly message: string; - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "message.received"; - } - | { - readonly data: { - readonly finishReason: EveAssistantStepFinishReason; - readonly message: string | null; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "message.completed"; - } - | { - readonly data: { - readonly reasoning: string; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "reasoning.completed"; - } - | { - readonly data: { - readonly result: EveJsonValue; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "result.completed"; - } - | { - readonly data: { - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.started"; - } - | { - readonly data: { - readonly finishReason: EveAssistantStepFinishReason; - readonly providerMetadata?: { - readonly gateway?: { - readonly generationId?: string; - }; - }; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - readonly usage?: { - readonly cacheReadTokens?: number; - readonly cacheWriteTokens?: number; - readonly costUsd?: number; - readonly inputTokens?: number; - readonly outputTokens?: number; - }; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.completed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.failed"; - } - | { - readonly data: { - readonly actions: readonly EveRuntimeActionRequest[]; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "actions.requested"; - } - | { - readonly data: { - readonly error?: EveActionResultError; - readonly result: EveRuntimeActionResult; - readonly sequence: number; - readonly stepIndex: number; - readonly status: EveActionResultStatus; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "action.result"; - } - | { - readonly data: { - readonly callId: string; - readonly childSessionId: string; - readonly name: string; - readonly remote?: { - readonly url?: string; - }; - readonly sequence: number; - readonly toolName?: string; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "subagent.called"; - } - | { - readonly data: { - readonly callId: string; - readonly error?: EveActionResultError; - readonly output?: EveJsonValue; - readonly sequence: number; - readonly status?: EveActionResultStatus; - readonly subagentName: string; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "subagent.completed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.failed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sessionId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.failed"; + readonly callId: string; + readonly input: unknown; + readonly toolName: string; + readonly type: "tool-call"; } | { - readonly data: { - readonly wait: "next-user-message"; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.waiting"; + readonly type: "tool-result"; } | { - readonly meta?: EveStreamEventMeta; - readonly type: "session.completed"; + readonly type: "tool-error"; }; -export interface EveHookDefinition { - readonly events?: { - readonly "*"?: ( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise; - readonly [eventType: string]: - | (( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise) - | undefined; - }; +export interface EveInstrumentationTurnStartedEvent { + readonly idempotencyKey: string; + readonly parentLineage?: EveInstrumentationParentLineage; + readonly parentTraceContext?: EveInstrumentationTraceContext; + readonly rootSessionId: string; + readonly sequence: number; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.started"; } -export interface EveInstrumentationSetupContext { - readonly agentName: string; +export interface EveInstrumentationTurnSettledEvent { + readonly idempotencyKey: string; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.cancelled" | "turn.completed"; } -type EveTextPart = { - readonly text: string; - readonly type: "text"; -}; - -type EveImagePart = { - readonly image: unknown; - readonly mediaType?: string; - readonly type: "image"; -}; - -type EveFilePart = { - readonly data: unknown; - readonly filename?: string; - readonly mediaType: string; - readonly type: "file"; -}; - -type EveReasoningPart = { - readonly text: string; - readonly type: "reasoning"; -}; - -type EveReasoningFilePart = { - readonly data: unknown; - readonly mediaType: string; - readonly type: "reasoning-file"; -}; - -type EveCustomPart = { - readonly kind: `${string}.${string}`; - readonly type: "custom"; -}; - -type EveToolCallPart = { - readonly input: unknown; - readonly providerExecuted?: boolean; - readonly toolCallId: string; - readonly toolName: string; - readonly type: "tool-call"; -}; - -type EveToolResultContentPart = - | EveTextPart - | EveFilePart - | { - readonly data: string; - readonly filename?: string; - readonly mediaType: string; - readonly type: "file-data"; - } - | { - readonly mediaType?: string; - readonly type: "file-url"; - readonly url: string; - } - | { - readonly fileId: string | Readonly>; - readonly type: "file-id" | "image-file-id"; - } - | { - readonly providerReference: Readonly>; - readonly type: "file-reference" | "image-file-reference"; - } - | { - readonly data: string; - readonly mediaType: string; - readonly type: "image-data"; - } - | { - readonly type: "image-url"; - readonly url: string; - } - | { readonly type: "custom" }; - -type EveToolResultOutput = - | { - readonly type: "text" | "error-text"; - readonly value: string; - } - | { - readonly type: "json" | "error-json"; - readonly value: EveJsonValue; - } - | { - readonly reason?: string; - readonly type: "execution-denied"; - } - | { - readonly type: "content"; - readonly value: readonly EveToolResultContentPart[]; - }; +export interface EveInstrumentationTurnFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.failed"; +} -type EveToolResultPart = { - readonly output: EveToolResultOutput; - readonly toolCallId: string; - readonly toolName: string; - readonly type: "tool-result"; -}; +export interface EveInstrumentationModelCallStartedEvent { + readonly idempotencyKey: string; + readonly input?: EveInstrumentationModelInput; + readonly model: { + readonly modelId: string; + readonly provider: string; + }; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.started"; +} -type EveToolApprovalRequest = { - readonly approvalId: string; - readonly isAutomatic?: boolean; - readonly signature?: string; - readonly toolCallId: string; - readonly type: "tool-approval-request"; -}; +export interface EveInstrumentationModelCallCompletedEvent { + readonly content?: readonly EveInstrumentationContentPart[]; + readonly finishReason: string; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.completed"; + readonly usage: EveInstrumentationUsage; +} -type EveToolApprovalResponse = { - readonly approvalId: string; - readonly approved: boolean; - readonly providerExecuted?: boolean; - readonly reason?: string; - readonly type: "tool-approval-response"; -}; +export interface EveInstrumentationModelCallFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.failed"; +} -export type EveModelMessageContentPart = - | EveTextPart - | EveImagePart - | EveFilePart - | EveReasoningPart - | EveReasoningFilePart - | EveCustomPart - | EveToolCallPart - | EveToolResultPart - | EveToolApprovalRequest - | EveToolApprovalResponse - | EveToolResultContentPart; +export interface EveInstrumentationStepAttemptCompletedEvent { + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "step.attempt.completed"; +} -export type EveSystemModelMessage = { - readonly content: string; - readonly role: "system"; -}; +export interface EveInstrumentationStepAttemptFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "step.attempt.failed"; +} -export type EveModelMessage = - | EveSystemModelMessage - | { - readonly content: - | string - | readonly (EveTextPart | EveImagePart | EveFilePart)[]; - readonly role: "user"; - } - | { - readonly content: - | string - | readonly ( - | EveTextPart - | EveCustomPart - | EveFilePart - | EveReasoningPart - | EveReasoningFilePart - | EveToolCallPart - | EveToolResultPart - | EveToolApprovalRequest - )[]; - readonly role: "assistant"; - } - | { - readonly content: readonly ( - | EveToolResultPart - | EveToolApprovalResponse - )[]; - readonly role: "tool"; - }; +export interface EveInstrumentationActionStartedEvent { + readonly callId: string; + readonly idempotencyKey: string; + readonly input?: unknown; + readonly name: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.started"; +} -export interface EveInstrumentationModelInput { - readonly instructions?: string | readonly EveSystemModelMessage[]; - readonly messages: readonly EveModelMessage[]; +export interface EveInstrumentationActionCompletedEvent { + readonly acceptedAtMs?: number; + readonly idempotencyKey: string; + readonly output: + | { readonly output?: unknown; readonly type: "result" } + | { readonly error?: unknown; readonly type: "error" }; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.completed"; } -export interface EveInstrumentationStepStartedEventInput { - readonly modelInput: EveInstrumentationModelInput; - readonly session: { - readonly id: string; - }; - readonly step: { - readonly index: number; - }; - readonly turn: { - readonly id: string; - readonly sequence: number; - }; +export interface EveInstrumentationActionFailedEvent { + readonly acceptedAtMs?: number; + readonly error?: unknown; + readonly errorCode?: string; + readonly idempotencyKey: string; + readonly outcome: "abandoned" | "cancelled" | "failed" | "rejected"; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.failed"; } +type EveInstrumentationHandler = ( + event: TEvent, + context: EveInstrumentationHandlerContext, +) => void | PromiseLike; + export interface EveInstrumentationDefinition { - readonly events?: { - readonly "step.started"?: ( - input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }; - readonly [eventType: string]: - | (( - input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }) - | undefined; + readonly capture: "content"; + readonly events: { + readonly "action.completed": EveInstrumentationHandler; + readonly "action.failed": EveInstrumentationHandler; + readonly "action.started": EveInstrumentationHandler; + readonly "model.call.completed": EveInstrumentationHandler; + readonly "model.call.failed": EveInstrumentationHandler; + readonly "model.call.started": EveInstrumentationHandler; + readonly "step.attempt.completed": EveInstrumentationHandler; + readonly "step.attempt.failed": EveInstrumentationHandler; + readonly "turn.cancelled": EveInstrumentationHandler; + readonly "turn.completed": EveInstrumentationHandler; + readonly "turn.failed": EveInstrumentationHandler; + readonly "turn.started": EveInstrumentationHandler; }; - readonly recordInputs?: boolean; - readonly recordOutputs?: boolean; - readonly setup?: (context: EveInstrumentationSetupContext) => void; + readonly flush: () => Promise; + readonly setup?: ( + context: EveInstrumentationSetupContext, + ) => void | PromiseLike; } diff --git a/js/src/wrappers/attachment-utils.ts b/js/src/wrappers/attachment-utils.ts index 7fb2b96f5..df7544fce 100644 --- a/js/src/wrappers/attachment-utils.ts +++ b/js/src/wrappers/attachment-utils.ts @@ -242,6 +242,26 @@ export function processInputAttachments(input: any): any { } } + // AI SDK tool output content format + if ( + (node.type === "image-data" || node.type === "file-data") && + node.data + ) { + const mediaType = node.mediaType || "application/octet-stream"; + const filename = + node.filename || + `output_${node.type === "image-data" ? "image" : "file"}_${attachmentIndex}.${getExtensionFromMediaType(mediaType)}`; + const attachment = toAttachment(node.data, mediaType, filename); + + if (attachment) { + attachmentIndex++; + return { + ...node, + data: attachment, + }; + } + } + const processed: Record = {}; for (const [key, value] of Object.entries(node)) { processed[key] = processNode(value); From df835ac838ec62378cb642586db867be242c395f Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:28:09 +0000 Subject: [PATCH 2/2] Update PR #2385 --- .changeset/remove-eve-pre-034.md | 5 +++++ js/src/instrumentation/plugins/eve-plugin.test.ts | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .changeset/remove-eve-pre-034.md diff --git a/.changeset/remove-eve-pre-034.md b/.changeset/remove-eve-pre-034.md new file mode 100644 index 000000000..1e3034506 --- /dev/null +++ b/.changeset/remove-eve-pre-034.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref(eve)!: Bump minimum eve version to `0.34.0` diff --git a/js/src/instrumentation/plugins/eve-plugin.test.ts b/js/src/instrumentation/plugins/eve-plugin.test.ts index f5db96192..38e15bef0 100644 --- a/js/src/instrumentation/plugins/eve-plugin.test.ts +++ b/js/src/instrumentation/plugins/eve-plugin.test.ts @@ -244,8 +244,6 @@ describe("braintrustEveInstrumentation", () => { await provider.events["turn.completed"]( { idempotencyKey: "turn:session-root:turn-0", - rootSessionId: "session-root", - sequence: 0, sessionId: "session-root", turnId: "turn-0", type: "turn.completed",