diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/index.ts index 813dc9341058..4bec21ba206b 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/index.ts @@ -21,16 +21,19 @@ const mockFetch: typeof fetch = async () => { status: 200, headers: { 'content-type': 'application/json' } }, ); -const client = Sentry.instrumentAnthropicAiClient(new Anthropic({ apiKey: 'mock-api-key', fetch: mockFetch })); - export default Sentry.withSentry( (env: Env) => ({ dsn: env.SENTRY_DSN, traceLifecycle: 'static', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: true, outputs: true } }, }), { async fetch(_request, _env, _ctx) { + // Wrapped in-request so the SDK is initialized when recording options are resolved, which is + // also the only place a real Worker can read its API key from `env`. + const client = Sentry.instrumentAnthropicAiClient(new Anthropic({ apiKey: 'mock-api-key', fetch: mockFetch })); + const response = await client.messages.create({ model: 'claude-3-haiku-20240307', messages: [{ role: 'user', content: 'What is the capital of France?' }], diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts index 4ba21f73c6c0..77e4269e40b6 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts @@ -1,11 +1,13 @@ import { expect, it } from 'vitest'; import { + GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, GEN_AI_REQUEST_MAX_TOKENS, GEN_AI_REQUEST_MODEL, GEN_AI_REQUEST_TEMPERATURE, GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_TEXT, GEN_AI_SYSTEM, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, @@ -45,6 +47,12 @@ it('traces a basic message creation request with the anthropic SDK', async ({ si [GEN_AI_REQUEST_MODEL]: { value: 'claude-3-haiku-20240307', type: 'string' }, [GEN_AI_REQUEST_TEMPERATURE]: { value: 0.7, type: 'double' }, [GEN_AI_REQUEST_MAX_TOKENS]: { value: 100, type: 'integer' }, + // collect only LLM input + [GEN_AI_INPUT_MESSAGES]: { + value: '[{"role":"user","content":"What is the capital of France?"}]', + type: 'string', + }, + [GEN_AI_RESPONSE_TEXT]: { value: 'Hello from Anthropic!', type: 'string' }, [GEN_AI_RESPONSE_ID]: { value: 'msg_mock123', type: 'string' }, [GEN_AI_RESPONSE_MODEL]: { value: 'claude-3-haiku-20240307', type: 'string' }, [GEN_AI_USAGE_INPUT_TOKENS]: { value: 10, type: 'integer' }, diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/index.ts index 56a650244de2..2b809fd9767a 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/index.ts @@ -39,16 +39,19 @@ globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { ); }) as typeof fetch; -const client = Sentry.instrumentGoogleGenAIClient(new GoogleGenAI({ apiKey: 'mock-api-key' })); - export default Sentry.withSentry( (env: Env) => ({ dsn: env.SENTRY_DSN, traceLifecycle: 'static', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: true } }, }), { async fetch(_request, _env, _ctx) { + // Wrapped in-request so the SDK is initialized when recording options are resolved, which is + // also the only place a real Worker can read its API key from `env`. + const client = Sentry.instrumentGoogleGenAIClient(new GoogleGenAI({ apiKey: 'mock-api-key' })); + // Test 1: chats.create and sendMessage flow const chat = client.chats.create({ model: 'gemini-1.5-pro', diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts index 0115791c3c22..e98f0a68bf0a 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts @@ -1,11 +1,14 @@ import { expect, it } from 'vitest'; import type { SerializedStreamedSpan } from '@sentry/core'; import { + GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, GEN_AI_REQUEST_MAX_TOKENS, GEN_AI_REQUEST_MODEL, GEN_AI_REQUEST_TEMPERATURE, GEN_AI_REQUEST_TOP_P, + GEN_AI_RESPONSE_TEXT, GEN_AI_SYSTEM, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, @@ -46,6 +49,9 @@ it('traces Google GenAI chat, generateContent, and embedContent calls', async ({ [GEN_AI_SYSTEM]: { value: 'google_genai', type: 'string' }, [GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: 'gemini-1.5-pro', type: 'string' }, + // collect LLM input and outputs (default true) + [GEN_AI_INPUT_MESSAGES]: { value: '[{"role":"user","content":"Tell me a joke"}]', type: 'string' }, + [GEN_AI_RESPONSE_TEXT]: { value: 'Hello from Google GenAI!', type: 'string' }, [GEN_AI_USAGE_INPUT_TOKENS]: { value: 8, type: 'integer' }, [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 12, type: 'integer' }, [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 20, type: 'integer' }, @@ -70,9 +76,14 @@ it('traces Google GenAI chat, generateContent, and embedContent calls', async ({ [GEN_AI_REQUEST_TEMPERATURE]: { value: 0.7, type: 'double' }, [GEN_AI_REQUEST_TOP_P]: { value: 0.9, type: 'double' }, [GEN_AI_REQUEST_MAX_TOKENS]: { value: 100, type: 'integer' }, + [GEN_AI_INPUT_MESSAGES]: { + value: '[{"role":"user","parts":[{"text":"What is the capital of France?"}]}]', + type: 'string', + }, [GEN_AI_USAGE_INPUT_TOKENS]: { value: 8, type: 'integer' }, [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 12, type: 'integer' }, [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 20, type: 'integer' }, + [GEN_AI_RESPONSE_TEXT]: { value: 'Hello from Google GenAI!', type: 'string' }, }, }); @@ -91,6 +102,7 @@ it('traces Google GenAI chat, generateContent, and embedContent calls', async ({ [GEN_AI_SYSTEM]: { value: 'google_genai', type: 'string' }, [GEN_AI_OPERATION_NAME]: { value: 'embeddings', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: 'text-embedding-004', type: 'string' }, + [GEN_AI_EMBEDDINGS_INPUT]: { value: 'Hello world', type: 'string' }, }, }); }) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/index.ts index bfb474400449..6b9cb7ee2bed 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/index.ts @@ -26,16 +26,19 @@ const mockFetch: typeof fetch = async () => { status: 200, headers: { 'content-type': 'application/json' } }, ); -const client = Sentry.instrumentOpenAiClient(new OpenAI({ apiKey: 'mock-api-key', fetch: mockFetch })); - export default Sentry.withSentry( (env: Env) => ({ dsn: env.SENTRY_DSN, traceLifecycle: 'static', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: true, outputs: false } }, }), { async fetch(_request, _env, _ctx) { + // Wrapped in-request so the SDK is initialized when recording options are resolved, which is + // also the only place a real Worker can read its API key from `env`. + const client = Sentry.instrumentOpenAiClient(new OpenAI({ apiKey: 'mock-api-key', fetch: mockFetch })); + const response = await client.chat.completions.create({ model: 'gpt-3.5-turbo', messages: [ diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts index e0e55cf7ec79..5675bfa7a080 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts @@ -1,5 +1,6 @@ import { expect, it } from 'vitest'; import { + GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_REQUEST_TEMPERATURE, @@ -7,6 +8,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_SYSTEM, + GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, @@ -44,6 +46,15 @@ it('traces a basic chat completion request with the openai SDK', async ({ signal [GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: 'gpt-3.5-turbo', type: 'string' }, [GEN_AI_REQUEST_TEMPERATURE]: { value: 0.7, type: 'double' }, + // collect only LLM input + [GEN_AI_SYSTEM_INSTRUCTIONS]: { + value: '[{"type":"text","content":"You are a helpful assistant."}]', + type: 'string', + }, + [GEN_AI_INPUT_MESSAGES]: { + value: '[{"role":"user","content":"What is the capital of France?"}]', + type: 'string', + }, [GEN_AI_RESPONSE_ID]: { value: 'chatcmpl-mock123', type: 'string' }, [GEN_AI_RESPONSE_MODEL]: { value: 'gpt-3.5-turbo', type: 'string' }, [GEN_AI_USAGE_INPUT_TOKENS]: { value: 10, type: 'integer' }, diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts index e0c2927d69bb..60755738eccb 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts @@ -6,16 +6,26 @@ interface Env { SENTRY_DSN: string; } -const ai = instrumentWorkersAiClient(new MockAi(), { recordInputs: false, recordOutputs: false }); +// Stands in for the `env.AI` binding: the binding object exists before the request, but is only +// reachable through `env` inside the handler. +const aiBinding = new MockAi(); export default Sentry.withSentry( (env: Env) => ({ dsn: env.SENTRY_DSN, traceLifecycle: 'static', tracesSampleRate: 1.0, + // Responses only. Asserting that prompts are *absent* is what makes this suite fail if the + // binding is ever wrapped before the SDK is initialized again, since the fallback collects both. + dataCollection: { genAI: { inputs: false, outputs: true } }, }), { async fetch(request) { + // Wrapped in-request, mirroring how `@sentry/cloudflare` instruments `env.AI` on first + // property access: after `withSentry` has initialized the SDK, so `dataCollection.genAI` + // is visible when recording options are resolved. + const ai = instrumentWorkersAiClient(aiBinding); + const url = new URL(request.url); if (url.pathname === '/error') { diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts index dbb6970b2e55..ec0b82cd69bc 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts @@ -1,10 +1,12 @@ import { GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MAX_TOKENS, GEN_AI_REQUEST_MODEL, GEN_AI_REQUEST_TEMPERATURE, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TEXT, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, @@ -44,6 +46,15 @@ it('traces a basic Workers AI text generation request', async ({ signal }) => { [GEN_AI_USAGE_INPUT_TOKENS]: { value: 12, type: 'integer' }, [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 7, type: 'integer' }, [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 19, type: 'integer' }, + // collect only output messages + [GEN_AI_OUTPUT_MESSAGES]: { + type: 'string', + value: '[{"role":"assistant","parts":[{"type":"text","content":"The capital of France is Paris."}]}]', + }, + [GEN_AI_RESPONSE_TEXT]: { + type: 'string', + value: 'The capital of France is Paris.', + }, }, }), ); @@ -80,6 +91,15 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) = [GEN_AI_USAGE_INPUT_TOKENS]: { value: 12, type: 'integer' }, [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 7, type: 'integer' }, [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 19, type: 'integer' }, + // collect only output + [GEN_AI_OUTPUT_MESSAGES]: { + type: 'string', + value: '[{"role":"assistant","parts":[{"type":"text","content":"The capital of France is Paris."}]}]', + }, + [GEN_AI_RESPONSE_TEXT]: { + type: 'string', + value: 'The capital of France is Paris.', + }, }, }), ); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-als/src/index.ts b/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-als/src/index.ts index 6b8cca2cbf31..aa4d046da5b3 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-als/src/index.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-als/src/index.ts @@ -9,6 +9,7 @@ export default Sentry.withSentry( environment: 'qa', tunnel: 'http://localhost:3031/', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, integrations: [Sentry.vercelAIIntegration()], }), { diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-compat/src/index.ts b/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-compat/src/index.ts index 60807d30fd61..daae29d1dd53 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-compat/src/index.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-compat/src/index.ts @@ -9,6 +9,7 @@ export default Sentry.withSentry( environment: 'qa', tunnel: 'http://localhost:3031/', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, integrations: [Sentry.vercelAIIntegration()], }), { diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs index d6a9efde7bae..14dbd52b3824 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs @@ -6,5 +6,6 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs index 170ad6f6a702..42052d281304 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs @@ -6,5 +6,6 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs index 53b9511a21f0..5e0b6fb5592f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs @@ -5,6 +5,7 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, traceLifecycle: 'stream', }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs index 53b9511a21f0..5e0b6fb5592f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs @@ -5,6 +5,7 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, traceLifecycle: 'stream', }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs index 170ad6f6a702..42052d281304 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs @@ -6,5 +6,6 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs index 170ad6f6a702..42052d281304 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs @@ -6,5 +6,6 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, }); diff --git a/packages/astro/test/server/middleware.test.ts b/packages/astro/test/server/middleware.test.ts index 4cac21a74845..6bb15c58629f 100644 --- a/packages/astro/test/server/middleware.test.ts +++ b/packages/astro/test/server/middleware.test.ts @@ -66,7 +66,7 @@ describe('sentryMiddleware', () => { httpBodies: [], urlQueryParams: true, graphQL: { document: true, variables: true }, - genAI: { inputs: false, outputs: false }, + genAI: { inputs: true, outputs: true }, databaseQueryData: true, stackFrameVariables: true, frameContextLines: 5, diff --git a/packages/core/src/integrations/mcp-server/index.ts b/packages/core/src/integrations/mcp-server/index.ts index df09714e404f..fd4c5168551a 100644 --- a/packages/core/src/integrations/mcp-server/index.ts +++ b/packages/core/src/integrations/mcp-server/index.ts @@ -64,8 +64,8 @@ export function wrapMcpServerWithSentry(mcpServerInstance: S, const genAI = client?.getDataCollectionOptions().genAI; const resolvedOptions: ResolvedMcpOptions = { - recordInputs: options?.recordInputs ?? genAI?.inputs ?? false, - recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? false, + recordInputs: options?.recordInputs ?? genAI?.inputs ?? true, + recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? true, }; fill(serverInstance, 'connect', originalConnect => { diff --git a/packages/core/src/tracing/ai/utils.ts b/packages/core/src/tracing/ai/utils.ts index 906b28a782f0..7c4ef9a563ce 100644 --- a/packages/core/src/tracing/ai/utils.ts +++ b/packages/core/src/tracing/ai/utils.ts @@ -46,14 +46,14 @@ export type InstrumentedMethodRegistry = Record /** * Resolves AI recording options by falling back to the client's `dataCollection.genAI` settings. - * Precedence: explicit option > dataCollection.genAI > sendDefaultPii > false + * Precedence: explicit option > dataCollection.genAI > true (genAI data collected by default) */ export function resolveAIRecordingOptions(options?: T): T & Required { const genAI = getClient()?.getDataCollectionOptions().genAI; return { ...options, - recordInputs: options?.recordInputs ?? genAI?.inputs ?? false, - recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? false, + recordInputs: options?.recordInputs ?? genAI?.inputs ?? true, + recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? true, } as T & Required; } diff --git a/packages/core/src/utils/data-collection/defaultPiiToCollectionOptions.ts b/packages/core/src/utils/data-collection/defaultPiiToCollectionOptions.ts index b70cddc267f4..a5bc57cb410a 100644 --- a/packages/core/src/utils/data-collection/defaultPiiToCollectionOptions.ts +++ b/packages/core/src/utils/data-collection/defaultPiiToCollectionOptions.ts @@ -31,7 +31,7 @@ export function defaultPiiToCollectionOptions(sendDefaultPii?: boolean): Resolve // The GraphQL document has literal values redacted at collection time, so it was historically // always attached regardless of `sendDefaultPii`; keep it on to preserve that behavior. graphQL: { document: true, variables: true }, - genAI: { inputs: false, outputs: false }, + genAI: { inputs: true, outputs: true }, // Database query values were only sent with `sendDefaultPii: true` (e.g. Supabase gated on it), // so map the legacy "off" state to `false`. databaseQueryData: false, diff --git a/packages/core/test/lib/integrations/mcp-server/transportInstrumentation.test.ts b/packages/core/test/lib/integrations/mcp-server/transportInstrumentation.test.ts index ef764b86f213..a7b00bb3b1c6 100644 --- a/packages/core/test/lib/integrations/mcp-server/transportInstrumentation.test.ts +++ b/packages/core/test/lib/integrations/mcp-server/transportInstrumentation.test.ts @@ -808,7 +808,8 @@ describe('MCP Server Transport Instrumentation', () => { ); }); - it('should NOT capture inputs/outputs when sendDefaultPii is false (legacy bridge)', async () => { + // todo: delete the following test once we remove sendDefaultPii + it('should capture inputs/outputs when sendDefaultPii is false (genAI collected by default)', async () => { getClientSpy.mockReturnValue(createTestClientWithSendDefaultPii(false)); const mockMcpServer = createMockMcpServer(); @@ -829,7 +830,7 @@ describe('MCP Server Transport Instrumentation', () => { expect(startInactiveSpanSpy).toHaveBeenCalledWith( expect.objectContaining({ - attributes: expect.not.objectContaining({ + attributes: expect.objectContaining({ 'mcp.request.argument.location': expect.anything(), }), }), diff --git a/packages/core/test/lib/tracing/ai/utils.test.ts b/packages/core/test/lib/tracing/ai/utils.test.ts index b46a047c3838..7a6286223dec 100644 --- a/packages/core/test/lib/tracing/ai/utils.test.ts +++ b/packages/core/test/lib/tracing/ai/utils.test.ts @@ -34,21 +34,22 @@ describe('resolveAIRecordingOptions', () => { client.init(); } - it('defaults to false when no client is set', () => { - expect(resolveAIRecordingOptions()).toEqual({ recordInputs: false, recordOutputs: false }); + it('defaults to true when no client is set', () => { + expect(resolveAIRecordingOptions()).toEqual({ recordInputs: true, recordOutputs: true }); }); - it('defaults to false when sendDefaultPii is false (bridge)', () => { + // todo: delete the following tests once we remove sendDefaultPii + it.skip('defaults to false when sendDefaultPii is false (bridge)', () => { setupWithSendDefaultPii(false); expect(resolveAIRecordingOptions()).toEqual({ recordInputs: false, recordOutputs: false }); }); - it('defaults to true when sendDefaultPii is true (bridge)', () => { + it.skip('defaults to true when sendDefaultPii is true (bridge)', () => { setupWithSendDefaultPii(true); expect(resolveAIRecordingOptions()).toEqual({ recordInputs: true, recordOutputs: true }); }); - it('explicit options override sendDefaultPii bridge', () => { + it.skip('explicit options override sendDefaultPii bridge', () => { setupWithSendDefaultPii(true); expect(resolveAIRecordingOptions({ recordInputs: false })).toEqual({ recordInputs: false, recordOutputs: true }); }); diff --git a/packages/core/test/lib/tracing/workers-ai.test.ts b/packages/core/test/lib/tracing/workers-ai.test.ts index 78e5dc79fe2f..e0fefb2ac2b4 100644 --- a/packages/core/test/lib/tracing/workers-ai.test.ts +++ b/packages/core/test/lib/tracing/workers-ai.test.ts @@ -1,13 +1,48 @@ +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_TEXT, + GEN_AI_SYSTEM_INSTRUCTIONS, +} from '@sentry/conventions/attributes'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { getCurrentScope, getGlobalScope, getIsolationScope, setCurrentClient, startSpan } from '../../../src'; +import type { Span } from '../../../src'; +import { + getCurrentScope, + getGlobalScope, + getIsolationScope, + SEMANTIC_ATTRIBUTE_SENTRY_OP, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE, + SEMANTIC_ATTRIBUTE_SENTRY_SOURCE, + setCurrentClient, + startSpan, +} from '../../../src'; import { addVercelAiProcessors } from '../../../src/tracing/vercel-ai'; import { AI_OPERATION_ID_ATTRIBUTE } from '../../../src/tracing/vercel-ai/vercel-ai-attributes'; import { instrumentWorkersAiClient } from '../../../src/tracing/workers-ai'; +import type { DataCollection } from '../../../src/types/datacollection'; import { _INTERNAL_clearAiProviderSkips } from '../../../src/utils/ai/providerSkip'; import { spanToJSON } from '../../../src/utils/spanUtils'; import { getDefaultTestClientOptions, TestClient } from '../../mocks/client'; +const MODEL = '@cf/meta/llama-3.1-8b-instruct'; + describe('instrumentWorkersAiClient', () => { + beforeEach(() => { + getCurrentScope().clear(); + getIsolationScope().clear(); + getGlobalScope().clear(); + }); + + afterEach(() => { + getCurrentScope().clear(); + getIsolationScope().clear(); + getGlobalScope().clear(); + }); + it('passes through non-run methods bound to the original client', () => { class FakeAi { #internal = 'secret'; @@ -29,17 +64,118 @@ describe('instrumentWorkersAiClient', () => { const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; const instrumented = instrumentWorkersAiClient(client); - const result = await instrumented.run('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' }); + const result = await instrumented.run(MODEL, { prompt: 'Hello' }); - expect(client.run).toHaveBeenCalledWith('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' }); + expect(client.run).toHaveBeenCalledWith(MODEL, { prompt: 'Hello' }); expect(result).toEqual({ response: 'Paris' }); }); + describe('message recording', () => { + /** + * Attributes recorded regardless of the `genAI` data collection settings. Sample rate and source + * are on here because the `run` call is the root span in these tests, with no active parent. + */ + const ALWAYS_RECORDED = { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.cloudflare.workers_ai', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', + [SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE]: 1, + [SEMANTIC_ATTRIBUTE_SENTRY_SOURCE]: 'custom', + [GEN_AI_PROVIDER_NAME]: 'cloudflare.workers_ai', + [GEN_AI_OPERATION_NAME]: 'chat', + [GEN_AI_REQUEST_MODEL]: MODEL, + }; + + /** Everything gated behind `recordInputs`. The system message is split into its own attribute. */ + const PROMPT_ATTRIBUTES = { + [GEN_AI_SYSTEM_INSTRUCTIONS]: '[{"type":"text","content":"Answer in one word."}]', + [GEN_AI_INPUT_MESSAGES]: '[{"role":"user","content":"Capital of France?"}]', + }; + + /** Everything gated behind `recordOutputs`. */ + const RESPONSE_ATTRIBUTES = { + [GEN_AI_OUTPUT_MESSAGES]: '[{"role":"assistant","parts":[{"type":"text","content":"Paris"}]}]', + [GEN_AI_RESPONSE_TEXT]: 'Paris', + }; + + function setupClient(dataCollection?: DataCollection): Span[] { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + dataCollection, + }), + ); + setCurrentClient(client); + client.init(); + + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + return endedSpans; + } + + // `@sentry/cloudflare` instruments the `env.AI` binding on first property access, which happens + // inside the request — after `withSentry` has initialized the SDK. These cases wrap in that same + // order so `dataCollection.genAI` is readable when recording options are resolved. + it.each([ + { + name: 'collects prompts and responses when dataCollection is unset', + dataCollection: undefined, + options: undefined, + expected: { ...ALWAYS_RECORDED, ...PROMPT_ATTRIBUTES, ...RESPONSE_ATTRIBUTES }, + }, + { + name: 'omits prompts and responses when dataCollection.genAI disables both', + dataCollection: { genAI: { inputs: false, outputs: false } }, + options: undefined, + expected: ALWAYS_RECORDED, + }, + { + name: 'records prompts only when dataCollection.genAI enables inputs alone', + dataCollection: { genAI: { inputs: true, outputs: false } }, + options: undefined, + expected: { ...ALWAYS_RECORDED, ...PROMPT_ATTRIBUTES }, + }, + { + name: 'records responses only when dataCollection.genAI enables outputs alone', + dataCollection: { genAI: { inputs: false, outputs: true } }, + options: undefined, + expected: { ...ALWAYS_RECORDED, ...RESPONSE_ATTRIBUTES }, + }, + { + name: 'defaults the unspecified half of dataCollection.genAI to collecting', + dataCollection: { genAI: { inputs: false } }, + options: undefined, + expected: { ...ALWAYS_RECORDED, ...RESPONSE_ATTRIBUTES }, + }, + { + name: 'prefers explicit recording options over dataCollection.genAI', + dataCollection: { genAI: { inputs: true, outputs: true } }, + options: { recordInputs: false, recordOutputs: false }, + expected: ALWAYS_RECORDED, + }, + ])('$name', async ({ dataCollection, options, expected }) => { + const endedSpans = setupClient(dataCollection); + + const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; + const instrumented = instrumentWorkersAiClient(client, options); + + await instrumented.run(MODEL, { + messages: [ + { role: 'system', content: 'Answer in one word.' }, + { role: 'user', content: 'Capital of France?' }, + ], + }); + + expect(endedSpans).toHaveLength(1); + expect(spanToJSON(endedSpans[0]!).data).toEqual(expected); + }); + }); + describe('when the Vercel AI SDK drives the binding', () => { let spans: string[]; /** Set up a client with the Vercel AI processors registered, recording every ended span. */ - function setupClient(): void { + function setupVercelAiClient(): void { getCurrentScope().clear(); getIsolationScope().clear(); getGlobalScope().clear(); @@ -55,7 +191,7 @@ describe('instrumentWorkersAiClient', () => { beforeEach(() => { _INTERNAL_clearAiProviderSkips(); - setupClient(); + setupVercelAiClient(); }); afterEach(() => { @@ -79,11 +215,11 @@ describe('instrumentWorkersAiClient', () => { const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; const instrumented = instrumentWorkersAiClient(client); - await withVercelAiModelCall(() => instrumented.run('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' })); + await withVercelAiModelCall(() => instrumented.run(MODEL, { prompt: 'Hello' })); // The call is forwarded, but no duplicate `gen_ai.chat` span is emitted — only the // Vercel AI model-call span remains. - expect(client.run).toHaveBeenCalledWith('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' }); + expect(client.run).toHaveBeenCalledWith(MODEL, { prompt: 'Hello' }); expect(spans).not.toContain('chat @cf/meta/llama-3.1-8b-instruct'); expect(spans).toEqual(['streamText.doStream']); }); @@ -92,9 +228,9 @@ describe('instrumentWorkersAiClient', () => { const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; const instrumented = instrumentWorkersAiClient(client); - await startSpan({ name: 'root' }, () => instrumented.run('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' })); + await startSpan({ name: 'root' }, () => instrumented.run(MODEL, { prompt: 'Hello' })); - expect(spans).toContain('chat @cf/meta/llama-3.1-8b-instruct'); + expect(spans).toEqual([`chat ${MODEL}`, 'root']); }); it('clears the skip between clients so a later isolate reuse is unaffected', async () => { @@ -102,17 +238,17 @@ describe('instrumentWorkersAiClient', () => { const instrumented = instrumentWorkersAiClient(client); // First request: the `ai` SDK runs and marks Workers AI as skipped. - await withVercelAiModelCall(() => instrumented.run('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' })); - expect(spans).not.toContain('chat @cf/meta/llama-3.1-8b-instruct'); + await withVercelAiModelCall(() => instrumented.run(MODEL, { prompt: 'Hello' })); + expect(spans).toEqual(['streamText.doStream']); // Second request on the same isolate: `_setupIntegrations` resets the registry, so a direct // `env.AI.run` call must get its span back. Without the reset this would stay suppressed. _INTERNAL_clearAiProviderSkips(); - setupClient(); + setupVercelAiClient(); - await startSpan({ name: 'root' }, () => instrumented.run('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' })); + await startSpan({ name: 'root' }, () => instrumented.run(MODEL, { prompt: 'Hello' })); - expect(spans).toContain('chat @cf/meta/llama-3.1-8b-instruct'); + expect(spans).toEqual([`chat ${MODEL}`, 'root']); }); }); }); diff --git a/packages/core/test/lib/utils/data-collection/defaultPiiToCollectionOptions.test.ts b/packages/core/test/lib/utils/data-collection/defaultPiiToCollectionOptions.test.ts index c76dc1dccb4b..23a5f38baeef 100644 --- a/packages/core/test/lib/utils/data-collection/defaultPiiToCollectionOptions.test.ts +++ b/packages/core/test/lib/utils/data-collection/defaultPiiToCollectionOptions.test.ts @@ -28,7 +28,7 @@ describe('defaultPiiToCollectionOptions', () => { httpBodies: [], urlQueryParams: { deny: ['forwarded', '-ip', 'remote-', 'via', '-user'] }, graphQL: { document: true, variables: true }, - genAI: { inputs: false, outputs: false }, + genAI: { inputs: true, outputs: true }, databaseQueryData: false, stackFrameVariables: true, frameContextLines: 7, @@ -46,7 +46,7 @@ describe('defaultPiiToCollectionOptions', () => { httpBodies: [], urlQueryParams: { deny: ['forwarded', '-ip', 'remote-', 'via', '-user'] }, graphQL: { document: true, variables: true }, - genAI: { inputs: false, outputs: false }, + genAI: { inputs: true, outputs: true }, databaseQueryData: false, stackFrameVariables: true, frameContextLines: 7, @@ -64,7 +64,7 @@ describe('defaultPiiToCollectionOptions', () => { httpBodies: [], urlQueryParams: { deny: ['forwarded', '-ip', 'remote-', 'via', '-user'] }, graphQL: { document: true, variables: true }, - genAI: { inputs: false, outputs: false }, + genAI: { inputs: true, outputs: true }, databaseQueryData: false, stackFrameVariables: true, frameContextLines: 7, diff --git a/packages/core/test/lib/utils/data-collection/resolveDataCollectionOptions.test.ts b/packages/core/test/lib/utils/data-collection/resolveDataCollectionOptions.test.ts index 5b2f9a22cee1..5521462d2334 100644 --- a/packages/core/test/lib/utils/data-collection/resolveDataCollectionOptions.test.ts +++ b/packages/core/test/lib/utils/data-collection/resolveDataCollectionOptions.test.ts @@ -22,7 +22,7 @@ describe('resolveDataCollectionOptions', () => { // sendDefaultPii undefined → restrictive bridge (backward compat; userInfo defaults to true only when dataCollection is set) expect(result.userInfo).toBe(false); expect(result.httpBodies).toEqual([]); - expect(result.genAI).toEqual({ inputs: false, outputs: false }); + expect(result.genAI).toEqual({ inputs: true, outputs: true }); // GraphQL documents are redacted at collection time, so they stay on to preserve legacy behavior. expect(result.graphQL).toEqual({ document: true, variables: true }); expect(result.databaseQueryData).toBe(false); @@ -60,7 +60,7 @@ describe('resolveDataCollectionOptions', () => { expect(result.userInfo).toBe(false); expect(result.httpBodies).toEqual([]); - expect(result.genAI).toEqual({ inputs: false, outputs: false }); + expect(result.genAI).toEqual({ inputs: true, outputs: true }); expect(result.graphQL).toEqual({ document: true, variables: true }); expect(result.databaseQueryData).toBe(false); }); diff --git a/packages/nuxt/test/runtime/hooks/wrapMiddlewareHandler.test.ts b/packages/nuxt/test/runtime/hooks/wrapMiddlewareHandler.test.ts index e84b6aeb461b..3a3c9bdca5c0 100644 --- a/packages/nuxt/test/runtime/hooks/wrapMiddlewareHandler.test.ts +++ b/packages/nuxt/test/runtime/hooks/wrapMiddlewareHandler.test.ts @@ -48,7 +48,7 @@ describe('wrapMiddlewareHandlerWithSentry', () => { httpBodies: [], urlQueryParams: true, graphQL: { document: true, variables: true }, - genAI: { inputs: false, outputs: false }, + genAI: { inputs: true, outputs: true }, databaseQueryData: true, stackFrameVariables: true, frameContextLines: 5,