Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,19 @@ const mockFetch: typeof fetch = async () =>
{ status: 200, headers: { 'content-type': 'application/json' } },
);

const client = Sentry.instrumentAnthropicAiClient(new Anthropic({ apiKey: 'mock-api-key', fetch: mockFetch }));

export default Sentry.withSentry(
(env: Env) => ({
dsn: env.SENTRY_DSN,
traceLifecycle: 'static',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: true, outputs: true } },
}),
{
async fetch(_request, _env, _ctx) {
// Wrapped in-request so the SDK is initialized when recording options are resolved, which is
// also the only place a real Worker can read its API key from `env`.
const client = Sentry.instrumentAnthropicAiClient(new Anthropic({ apiKey: 'mock-api-key', fetch: mockFetch }));

const response = await client.messages.create({
model: 'claude-3-haiku-20240307',
messages: [{ role: 'user', content: 'What is the capital of France?' }],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { expect, it } from 'vitest';
import {
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_REQUEST_MAX_TOKENS,
GEN_AI_REQUEST_MODEL,
GEN_AI_REQUEST_TEMPERATURE,
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_TEXT,
GEN_AI_SYSTEM,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
Expand Down Expand Up @@ -45,6 +47,12 @@ it('traces a basic message creation request with the anthropic SDK', async ({ si
[GEN_AI_REQUEST_MODEL]: { value: 'claude-3-haiku-20240307', type: 'string' },
[GEN_AI_REQUEST_TEMPERATURE]: { value: 0.7, type: 'double' },
[GEN_AI_REQUEST_MAX_TOKENS]: { value: 100, type: 'integer' },
// collect only LLM input
[GEN_AI_INPUT_MESSAGES]: {
value: '[{"role":"user","content":"What is the capital of France?"}]',
type: 'string',
},
[GEN_AI_RESPONSE_TEXT]: { value: 'Hello from Anthropic!', type: 'string' },
[GEN_AI_RESPONSE_ID]: { value: 'msg_mock123', type: 'string' },
[GEN_AI_RESPONSE_MODEL]: { value: 'claude-3-haiku-20240307', type: 'string' },
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 10, type: 'integer' },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,19 @@ globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
);
}) as typeof fetch;

const client = Sentry.instrumentGoogleGenAIClient(new GoogleGenAI({ apiKey: 'mock-api-key' }));

export default Sentry.withSentry(
(env: Env) => ({
dsn: env.SENTRY_DSN,
traceLifecycle: 'static',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: true } },
}),
{
async fetch(_request, _env, _ctx) {
// Wrapped in-request so the SDK is initialized when recording options are resolved, which is
// also the only place a real Worker can read its API key from `env`.
const client = Sentry.instrumentGoogleGenAIClient(new GoogleGenAI({ apiKey: 'mock-api-key' }));

// Test 1: chats.create and sendMessage flow
const chat = client.chats.create({
model: 'gemini-1.5-pro',
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import { expect, it } from 'vitest';
import type { SerializedStreamedSpan } from '@sentry/core';
import {
GEN_AI_EMBEDDINGS_INPUT,
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_REQUEST_MAX_TOKENS,
GEN_AI_REQUEST_MODEL,
GEN_AI_REQUEST_TEMPERATURE,
GEN_AI_REQUEST_TOP_P,
GEN_AI_RESPONSE_TEXT,
GEN_AI_SYSTEM,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
Expand Down Expand Up @@ -46,6 +49,9 @@ it('traces Google GenAI chat, generateContent, and embedContent calls', async ({
[GEN_AI_SYSTEM]: { value: 'google_genai', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: 'gemini-1.5-pro', type: 'string' },
// collect LLM input and outputs (default true)
[GEN_AI_INPUT_MESSAGES]: { value: '[{"role":"user","content":"Tell me a joke"}]', type: 'string' },
[GEN_AI_RESPONSE_TEXT]: { value: 'Hello from Google GenAI!', type: 'string' },
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 8, type: 'integer' },
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 12, type: 'integer' },
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 20, type: 'integer' },
Expand All @@ -70,9 +76,14 @@ it('traces Google GenAI chat, generateContent, and embedContent calls', async ({
[GEN_AI_REQUEST_TEMPERATURE]: { value: 0.7, type: 'double' },
[GEN_AI_REQUEST_TOP_P]: { value: 0.9, type: 'double' },
[GEN_AI_REQUEST_MAX_TOKENS]: { value: 100, type: 'integer' },
[GEN_AI_INPUT_MESSAGES]: {
value: '[{"role":"user","parts":[{"text":"What is the capital of France?"}]}]',
type: 'string',
},
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 8, type: 'integer' },
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 12, type: 'integer' },
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 20, type: 'integer' },
[GEN_AI_RESPONSE_TEXT]: { value: 'Hello from Google GenAI!', type: 'string' },
},
});

Expand All @@ -91,6 +102,7 @@ it('traces Google GenAI chat, generateContent, and embedContent calls', async ({
[GEN_AI_SYSTEM]: { value: 'google_genai', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'embeddings', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: 'text-embedding-004', type: 'string' },
[GEN_AI_EMBEDDINGS_INPUT]: { value: 'Hello world', type: 'string' },
},
});
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,19 @@ const mockFetch: typeof fetch = async () =>
{ status: 200, headers: { 'content-type': 'application/json' } },
);

const client = Sentry.instrumentOpenAiClient(new OpenAI({ apiKey: 'mock-api-key', fetch: mockFetch }));

export default Sentry.withSentry(
(env: Env) => ({
dsn: env.SENTRY_DSN,
traceLifecycle: 'static',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: true, outputs: false } },
}),
{
async fetch(_request, _env, _ctx) {
// Wrapped in-request so the SDK is initialized when recording options are resolved, which is
// also the only place a real Worker can read its API key from `env`.
const client = Sentry.instrumentOpenAiClient(new OpenAI({ apiKey: 'mock-api-key', fetch: mockFetch }));

const response = await client.chat.completions.create({
model: 'gpt-3.5-turbo',
messages: [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import { expect, it } from 'vitest';
import {
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_REQUEST_MODEL,
GEN_AI_REQUEST_TEMPERATURE,
GEN_AI_RESPONSE_FINISH_REASONS,
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_SYSTEM,
GEN_AI_SYSTEM_INSTRUCTIONS,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
Expand Down Expand Up @@ -44,6 +46,15 @@ it('traces a basic chat completion request with the openai SDK', async ({ signal
[GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: 'gpt-3.5-turbo', type: 'string' },
[GEN_AI_REQUEST_TEMPERATURE]: { value: 0.7, type: 'double' },
// collect only LLM input
[GEN_AI_SYSTEM_INSTRUCTIONS]: {
value: '[{"type":"text","content":"You are a helpful assistant."}]',
type: 'string',
},
[GEN_AI_INPUT_MESSAGES]: {
value: '[{"role":"user","content":"What is the capital of France?"}]',
type: 'string',
},
[GEN_AI_RESPONSE_ID]: { value: 'chatcmpl-mock123', type: 'string' },
[GEN_AI_RESPONSE_MODEL]: { value: 'gpt-3.5-turbo', type: 'string' },
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 10, type: 'integer' },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,26 @@ interface Env {
SENTRY_DSN: string;
}

const ai = instrumentWorkersAiClient(new MockAi(), { recordInputs: false, recordOutputs: false });
// Stands in for the `env.AI` binding: the binding object exists before the request, but is only
// reachable through `env` inside the handler.
const aiBinding = new MockAi();

export default Sentry.withSentry(
(env: Env) => ({
dsn: env.SENTRY_DSN,
traceLifecycle: 'static',
tracesSampleRate: 1.0,
// Responses only. Asserting that prompts are *absent* is what makes this suite fail if the
// binding is ever wrapped before the SDK is initialized again, since the fallback collects both.
dataCollection: { genAI: { inputs: false, outputs: true } },
}),
{
async fetch(request) {
// Wrapped in-request, mirroring how `@sentry/cloudflare` instruments `env.AI` on first
// property access: after `withSentry` has initialized the SDK, so `dataCollection.genAI`
// is visible when recording options are resolved.
const ai = instrumentWorkersAiClient(aiBinding);

const url = new URL(request.url);

if (url.pathname === '/error') {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import {
GEN_AI_OPERATION_NAME,
GEN_AI_OUTPUT_MESSAGES,
GEN_AI_PROVIDER_NAME,
GEN_AI_REQUEST_MAX_TOKENS,
GEN_AI_REQUEST_MODEL,
GEN_AI_REQUEST_TEMPERATURE,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TEXT,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
Expand Down Expand Up @@ -44,6 +46,15 @@ it('traces a basic Workers AI text generation request', async ({ signal }) => {
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 12, type: 'integer' },
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 7, type: 'integer' },
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 19, type: 'integer' },
// collect only output messages
[GEN_AI_OUTPUT_MESSAGES]: {
type: 'string',
value: '[{"role":"assistant","parts":[{"type":"text","content":"The capital of France is Paris."}]}]',
},
[GEN_AI_RESPONSE_TEXT]: {
type: 'string',
value: 'The capital of France is Paris.',
},
},
}),
);
Expand Down Expand Up @@ -80,6 +91,15 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) =
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 12, type: 'integer' },
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 7, type: 'integer' },
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 19, type: 'integer' },
// collect only output
[GEN_AI_OUTPUT_MESSAGES]: {
type: 'string',
value: '[{"role":"assistant","parts":[{"type":"text","content":"The capital of France is Paris."}]}]',
},
[GEN_AI_RESPONSE_TEXT]: {
type: 'string',
value: 'The capital of France is Paris.',
},
},
}),
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export default Sentry.withSentry(
environment: 'qa',
tunnel: 'http://localhost:3031/',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
integrations: [Sentry.vercelAIIntegration()],
}),
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export default Sentry.withSentry(
environment: 'qa',
tunnel: 'http://localhost:3031/',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
integrations: [Sentry.vercelAIIntegration()],
}),
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
transport: loggingTransport,
traceLifecycle: 'stream',
});
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
transport: loggingTransport,
traceLifecycle: 'stream',
});
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
transport: loggingTransport,
});
2 changes: 1 addition & 1 deletion packages/astro/test/server/middleware.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ describe('sentryMiddleware', () => {
httpBodies: [],
urlQueryParams: true,
graphQL: { document: true, variables: true },
genAI: { inputs: false, outputs: false },
genAI: { inputs: true, outputs: true },
databaseQueryData: true,
stackFrameVariables: true,
frameContextLines: 5,
Expand Down
4 changes: 2 additions & 2 deletions packages/core/src/integrations/mcp-server/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ export function wrapMcpServerWithSentry<S extends object>(mcpServerInstance: S,
const genAI = client?.getDataCollectionOptions().genAI;

const resolvedOptions: ResolvedMcpOptions = {
recordInputs: options?.recordInputs ?? genAI?.inputs ?? false,
recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? false,
recordInputs: options?.recordInputs ?? genAI?.inputs ?? true,
recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? true,
};

fill(serverInstance, 'connect', originalConnect => {
Expand Down
6 changes: 3 additions & 3 deletions packages/core/src/tracing/ai/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,14 @@ export type InstrumentedMethodRegistry = Record<string, InstrumentedMethodEntry>

/**
* Resolves AI recording options by falling back to the client's `dataCollection.genAI` settings.
* Precedence: explicit option > dataCollection.genAI > sendDefaultPii > false
* Precedence: explicit option > dataCollection.genAI > true (genAI data collected by default)
*/
export function resolveAIRecordingOptions<T extends AIRecordingOptions>(options?: T): T & Required<AIRecordingOptions> {
const genAI = getClient()?.getDataCollectionOptions().genAI;
return {
Comment thread
s1gr1d marked this conversation as resolved.
...options,
recordInputs: options?.recordInputs ?? genAI?.inputs ?? false,
recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? false,
recordInputs: options?.recordInputs ?? genAI?.inputs ?? true,
recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? true,
} as T & Required<AIRecordingOptions>;
}
Comment thread
s1gr1d marked this conversation as resolved.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export function defaultPiiToCollectionOptions(sendDefaultPii?: boolean): Resolve
// The GraphQL document has literal values redacted at collection time, so it was historically
// always attached regardless of `sendDefaultPii`; keep it on to preserve that behavior.
graphQL: { document: true, variables: true },
genAI: { inputs: false, outputs: false },
genAI: { inputs: true, outputs: true },
// Database query values were only sent with `sendDefaultPii: true` (e.g. Supabase gated on it),
// so map the legacy "off" state to `false`.
databaseQueryData: false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -808,7 +808,8 @@ describe('MCP Server Transport Instrumentation', () => {
);
});

it('should NOT capture inputs/outputs when sendDefaultPii is false (legacy bridge)', async () => {
// todo: delete the following test once we remove sendDefaultPii
it('should capture inputs/outputs when sendDefaultPii is false (genAI collected by default)', async () => {
getClientSpy.mockReturnValue(createTestClientWithSendDefaultPii(false));

const mockMcpServer = createMockMcpServer();
Expand All @@ -829,7 +830,7 @@ describe('MCP Server Transport Instrumentation', () => {

expect(startInactiveSpanSpy).toHaveBeenCalledWith(
expect.objectContaining({
attributes: expect.not.objectContaining({
attributes: expect.objectContaining({
'mcp.request.argument.location': expect.anything(),
}),
}),
Expand Down
11 changes: 6 additions & 5 deletions packages/core/test/lib/tracing/ai/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,21 +34,22 @@ describe('resolveAIRecordingOptions', () => {
client.init();
}

it('defaults to false when no client is set', () => {
expect(resolveAIRecordingOptions()).toEqual({ recordInputs: false, recordOutputs: false });
it('defaults to true when no client is set', () => {
expect(resolveAIRecordingOptions()).toEqual({ recordInputs: true, recordOutputs: true });
});

it('defaults to false when sendDefaultPii is false (bridge)', () => {
// todo: delete the following tests once we remove sendDefaultPii
it.skip('defaults to false when sendDefaultPii is false (bridge)', () => {
setupWithSendDefaultPii(false);
expect(resolveAIRecordingOptions()).toEqual({ recordInputs: false, recordOutputs: false });
});

it('defaults to true when sendDefaultPii is true (bridge)', () => {
it.skip('defaults to true when sendDefaultPii is true (bridge)', () => {
setupWithSendDefaultPii(true);
expect(resolveAIRecordingOptions()).toEqual({ recordInputs: true, recordOutputs: true });
});

it('explicit options override sendDefaultPii bridge', () => {
it.skip('explicit options override sendDefaultPii bridge', () => {
setupWithSendDefaultPii(true);
expect(resolveAIRecordingOptions({ recordInputs: false })).toEqual({ recordInputs: false, recordOutputs: true });
});
Expand Down
Loading
Loading