diff --git a/.changeset/openai-batch-api-instrumentation.md b/.changeset/openai-batch-api-instrumentation.md new file mode 100644 index 000000000..fb7122b69 --- /dev/null +++ b/.changeset/openai-batch-api-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add OpenAI Batch API Instrumentation diff --git a/.changeset/update-platform-types.md b/.changeset/update-platform-types.md new file mode 100644 index 000000000..7de80491b --- /dev/null +++ b/.changeset/update-platform-types.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +chore: Update platform types diff --git a/AGENTS.md b/AGENTS.md index 6338e8830..91b5f76f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,10 @@ mise install # Install toolchain and dependencies pnpm run build # Build all workspace packages (from repo root) ``` +## Public API + +Keep public exports minimal. Generally, export only the requested runtime APIs and do not export types unless explicitly requested. + ## Instrumentation Use the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch. diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json index 49e7bb1d8..d6b8c55fe 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +786,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +896,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1006,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1113,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1325,248 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 16, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 62 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt index cbce77cc0..6c78bd0f3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 16, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 62 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json index 49e7bb1d8..95937b572 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +786,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +896,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1006,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1113,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1325,248 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 22, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 68 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt index cbce77cc0..87875d5b7 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 22, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 68 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json index 49e7bb1d8..8385ab9ee 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +778,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +888,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +998,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1105,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1317,256 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives a total of 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives a total of 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 22, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 68 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt index cbce77cc0..eba4aeaed 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "Adding 20 and 4 gives a total of 24.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "Adding 20 and 4 gives a total of 24.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 22, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 68 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json index 49e7bb1d8..1be92a331 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +778,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +888,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +998,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1105,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1317,256 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Twenty plus four equals twenty-four.", + "value": 24 + }, + "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Twenty plus four equals twenty-four.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 16, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 62 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt index cbce77cc0..8050c979e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "Twenty plus four equals twenty-four.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "Twenty plus four equals twenty-four.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 16, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 62 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json index 49e7bb1d8..ed152f17e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +786,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +896,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1006,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1113,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1325,248 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 26, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 72 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt index cbce77cc0..331725bf9 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.11.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 26, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 72 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json index 49e7bb1d8..08ffe28d3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +786,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +896,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1006,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1113,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1325,248 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 35, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 81 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt index cbce77cc0..dc15bc569 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.23.2", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 35, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 81 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json index 49e7bb1d8..ea5fc9fae 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +778,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +888,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +998,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1105,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1317,256 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 35, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 81 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt index cbce77cc0..4a83f4dc2 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.23.2", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 35, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 81 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json index 49e7bb1d8..097c2c455 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +778,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +888,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +998,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1105,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1317,256 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", + "value": 24 + }, + "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 26, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 72 } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "responses-parse", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-parse" + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt index cbce77cc0..54d9a2ee3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt @@ -1,546 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.11.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 26, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 72 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream-fixture" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "sync-stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: { - "kind": "text" - } - output: [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json index a9eea3198..bf45c4c47 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +731,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +787,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +796,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +818,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +931,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1041,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1151,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1258,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1364,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1470,126 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 17, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 63 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1601,192 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 204, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 337 + } + } + ], + "metadata": { + "operation": "responses-compact", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" } } ], - "input": { - "kind": "undefined" + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 }, - "output": null, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-compact" + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt index cea5718a8..443e4a96e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt @@ -1,655 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.25.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 17, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 63 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 204, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 337 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-image-attachment" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-fixture" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "sync-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" - │ ], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content_kind": "blocks", - "role": "user" - }, - { - "content_kind": "blocks", - "role": "assistant" - } - ] - output: [ - { - "content_types": [ - "input_text" - ], - "json_keys": [], - "role": "user", - "status": "completed", - "type": "message" - }, - { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, - "type": "compaction" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json index a9eea3198..2dcad18c0 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +731,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +787,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +796,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +818,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +931,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1041,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1151,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1258,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1364,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1470,126 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 24, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 70 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1601,192 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 235, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 368 + } + } + ], + "metadata": { + "operation": "responses-compact", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.49.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" } } ], - "input": { - "kind": "undefined" + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 }, - "output": null, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-compact" + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt index cea5718a8..b754f6b7c 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt @@ -1,655 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.49.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 70 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 235, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 368 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-image-attachment" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-fixture" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "sync-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" - │ ], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content_kind": "blocks", - "role": "user" - }, - { - "content_kind": "blocks", - "role": "assistant" - } - ] - output: [ - { - "content_types": [ - "input_text" - ], - "json_keys": [], - "role": "user", - "status": "completed", - "type": "message" - }, - { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, - "type": "compaction" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json index a9eea3198..182d9da94 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +734,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +779,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +788,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +810,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +923,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1033,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1143,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1250,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1356,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1462,134 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 24, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 70 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1601,192 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 235, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 368 + } + } + ], + "metadata": { + "operation": "responses-compact", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.49.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" } } ], - "input": { - "kind": "undefined" + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 }, - "output": null, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-compact" + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt index cea5718a8..c396bec2a 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt @@ -1,655 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.49.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "parsed": { +│ │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_parsed": { +│ │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 70 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 235, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 368 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-image-attachment" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-fixture" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "sync-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" - │ ], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content_kind": "blocks", - "role": "user" - }, - { - "content_kind": "blocks", - "role": "assistant" - } - ] - output: [ - { - "content_types": [ - "input_text" - ], - "json_keys": [], - "role": "user", - "status": "completed", - "type": "message" - }, - { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, - "type": "compaction" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json index a9eea3198..586b97eaf 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +734,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +779,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +788,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +810,110 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +923,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1033,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1143,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1250,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1356,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1462,134 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "20 plus 4 equals 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "20 plus 4 equals 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 17, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 63 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1601,192 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 204, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 337 + } + } + ], + "metadata": { + "operation": "responses-compact", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" } } ], - "input": { - "kind": "undefined" + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 }, - "output": null, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], "metadata": { - "operation": "responses-compact" + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt index cea5718a8..1c6d3fe8d 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt @@ -1,655 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.25.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "parsed": { +│ │ "reasoning": "20 plus 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_parsed": { +│ │ "reasoning": "20 plus 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 17, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 63 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 204, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 337 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-image-attachment" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "blocks", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "chat-tool" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null + │ } + │ ] │ metadata: { - │ "operation": "stream" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-tool" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "stream-fixture" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "sync-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content_kind": "string", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "json_keys": [], - │ "role": "assistant" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } - │ output: null - │ metadata: { - │ "operation": "embeddings" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "moderations" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-with-response" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-create-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-stream-partial" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null - │ metadata: { - │ "operation": "responses-parse" - │ } - │ metrics: { - │ "has_time_to_first_token": false - │ } - │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } - │ output: [ - │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" - │ ], - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ metrics: { - │ "has_time_to_first_token": true - │ } - └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } - metrics: { - "has_time_to_first_token": false + error: "Batch fixture request failed" + context: { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content_kind": "blocks", - "role": "user" - }, - { - "content_kind": "blocks", - "role": "assistant" - } - ] - output: [ - { - "content_types": [ - "input_text" - ], - "json_keys": [], - "role": "user", - "status": "completed", - "type": "message" - }, - { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, - "type": "compaction" - } - ] - metadata: { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - } - metrics: { - "has_time_to_first_token": true - } diff --git a/e2e/scenarios/openai-instrumentation/assertions.ts b/e2e/scenarios/openai-instrumentation/assertions.ts index f9684e4f2..5f7439e41 100644 --- a/e2e/scenarios/openai-instrumentation/assertions.ts +++ b/e2e/scenarios/openai-instrumentation/assertions.ts @@ -2,7 +2,6 @@ import { existsSync } from "node:fs"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; import { beforeAll, describe, expect, test } from "vitest"; -import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { @@ -45,7 +44,9 @@ type RelevantEvent = { type OperationSpec = { childNames: readonly string[]; + nestedChildNames?: readonly string[]; expectsOutput: boolean; + expectsModel?: boolean; expectsTimeToFirstToken: boolean; minOpenAIMajorVersion?: number; name: string; @@ -55,6 +56,12 @@ type OperationSpec = { validate?: (span: CapturedLogEvent | undefined) => void; }; +const EXPECTED_BATCH_OUTPUTS = new Map([ + ["Reply with exactly ALPHA.", "ALPHA"], + ["Reply with exactly BRAVO.", "BRAVO"], + ["Reply with exactly CHARLIE.", undefined], +]); + function asRecord(value: unknown): Record | undefined { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -233,6 +240,20 @@ const OPERATION_SPECS: readonly OperationSpec[] = [ expect(Array.isArray(span?.output)).toBe(true); }, }, + { + childNames: ["openai.batch"], + nestedChildNames: ["Chat Completion"], + expectsModel: false, + expectsOutput: false, + expectsTimeToFirstToken: false, + name: "openai-batch-operation", + operation: "batch", + testName: "captures resumable OpenAI Batch task and LLM spans", + validate: (span) => { + expect(span?.input).toBeUndefined(); + expect(span?.output).toBeUndefined(); + }, + }, { childNames: ["openai.responses.create"], expectsOutput: true, @@ -320,245 +341,22 @@ function getOperationSpecs(version: string): OperationSpec[] { ); } -function pickMetadata( - metadata: Record | undefined, - keys: string[], -): Json { - if (!metadata) { - return null; - } - - const picked = Object.fromEntries( - keys.flatMap((key) => { - const value = metadata[key]; - if (value === undefined) { - return []; - } - - return [ - [ - key, - key === "openaiSdkVersion" && typeof value === "string" - ? "" - : (value as Json), - ], - ]; - }), - ); - - return Object.keys(picked).length > 0 ? (picked as Json) : null; -} - -function isRecord(value: Json | undefined): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function summarizeMetricPresence(metrics: Json): Json { - if (!isRecord(metrics)) { - return null; - } - - return { - has_time_to_first_token: typeof metrics.time_to_first_token === "number", - } satisfies Json; -} - -function jsonKeysFromText(value: unknown): string[] { - if (typeof value !== "string") { - return []; - } - - try { - const parsed = JSON.parse(value) as unknown; - if ( - typeof parsed !== "object" || - parsed === null || - Array.isArray(parsed) - ) { - return []; - } - - return Object.keys(parsed).sort(); - } catch { - return []; - } -} - -function summarizeInput(input: Json): Json { - if (typeof input === "string") { - return { - kind: "text", - } satisfies Json; - } - - if (!Array.isArray(input)) { - return input === null ? null : ({ kind: typeof input } satisfies Json); - } - - return input.map((message) => { - if (!isRecord(message as Json)) { - return { kind: typeof message } satisfies Json; - } - - return { - content_kind: Array.isArray(message.content) - ? "blocks" - : typeof message.content, - role: message.role ?? null, - } satisfies Json; - }) satisfies Json; -} - -function summarizeChatOutput(output: Json): Json { - if (!Array.isArray(output)) { - return null; - } - - return output.map((choice) => { - if (!isRecord(choice as Json) || !isRecord(choice.message as Json)) { - return null; - } - - return { - json_keys: jsonKeysFromText(choice.message.content), - role: choice.message.role ?? null, - } satisfies Json; - }) satisfies Json; -} - -function summarizeResponsesOutput( - output: Json, - options?: { - // For normalization across SDK versions - dropEmptyOutputTextMessages?: boolean; - }, -): Json { - if (!Array.isArray(output)) { - return null; - } - - const summaries = output.map((item) => { - if (!isRecord(item as Json)) { - return null; - } - - const content = Array.isArray(item.content) ? item.content : []; - const contentTypes = content.flatMap((entry) => - isRecord(entry as Json) && typeof entry.type === "string" - ? [entry.type] - : [], - ); - const jsonKeys = content.flatMap((entry) => - isRecord(entry as Json) ? jsonKeysFromText(entry.text) : [], - ); - - return { - content_types: contentTypes, - json_keys: [...new Set(jsonKeys)].sort(), - role: item.role ?? null, - status: item.status ?? null, - type: item.type ?? null, - } satisfies Json; - }); - - const filtered = options?.dropEmptyOutputTextMessages - ? summaries.filter((item) => { - if (!isRecord(item as Json)) { - return true; - } - - return !( - item.role === "assistant" && - item.status === "completed" && - item.type === "message" && - Array.isArray(item.content_types) && - item.content_types.length === 1 && - item.content_types[0] === "output_text" && - Array.isArray(item.json_keys) && - item.json_keys.length === 0 - ); - }) - : summaries; - - // Deduplicate identical items — the Responses API occasionally returns - // duplicate output entries (e.g., two identical "message" items when - // streaming), which would cause non-deterministic snapshot failures. - const seen = new Set(); - const deduped: Json[] = []; - - for (const summarized of filtered) { - const key = JSON.stringify(summarized); - if (!seen.has(key)) { - seen.add(key); - deduped.push(summarized); +function findOpenAISpan( + events: CapturedLogEvent[], + parentId: string | undefined, + names: readonly string[], +) { + for (const name of names) { + const spans = findChildSpans(events, name, parentId); + if (spans.length > 0) { + return spans.find((span) => span.output !== undefined) ?? spans.at(-1); } } - return deduped; -} - -function summarizeOutput(name: string, output: Json): Json { - if (name === "Chat Completion") { - return summarizeChatOutput(output); - } - - if (name === "Embedding") { - return isRecord(output) - ? ({ - embedding_length: output.embedding_length ?? null, - } satisfies Json) - : null; - } - - if (name === "Moderation") { - return Array.isArray(output) - ? ({ - flagged_count: output.filter( - (entry) => isRecord(entry as Json) && entry.flagged === true, - ).length, - result_count: output.length, - } satisfies Json) - : null; - } - - if ( - name === "openai.responses.create" || - name === "openai.responses.compact" - ) { - return summarizeResponsesOutput(output); - } - - if (name === "openai.responses.parse") { - return summarizeResponsesOutput(output, { - dropEmptyOutputTextMessages: true, - }); - } - - return output === null || output === undefined - ? null - : ({ kind: typeof output } satisfies Json); -} - -function summarizeOpenAIPayload( - event: CapturedLogEvent, - summaryName: string | undefined, -): Json { - const name = summaryName ?? event.span.name ?? ""; - - return { - input: summarizeInput(event.input as Json), - metadata: pickMetadata( - event.row.metadata as Record | undefined, - ["model", "openaiSdkVersion", "operation", "provider", "scenario"], - ), - metrics: summarizeMetricPresence(event.metrics as Json), - name: name || null, - output: summarizeOutput(name, event.output as Json), - type: event.span.type ?? null, - } satisfies Json; + return undefined; } -function findOpenAISpan( +function findOpenAISpans( events: CapturedLogEvent[], parentId: string | undefined, names: readonly string[], @@ -566,11 +364,17 @@ function findOpenAISpan( for (const name of names) { const spans = findChildSpans(events, name, parentId); if (spans.length > 0) { - return spans.find((span) => span.output !== undefined) ?? spans.at(-1); + return spans; } } - return undefined; + return []; +} + +function batchPrompt(span: CapturedLogEvent): string { + const firstMessage = Array.isArray(span.input) ? span.input[0] : undefined; + const content = asRecord(firstMessage)?.content; + return typeof content === "string" ? content : ""; } function buildRelevantEvents( @@ -584,10 +388,27 @@ function buildRelevantEvents( for (const spec of operationSpecs) { const operation = findLatestSpan(events, spec.name)!; relevantEvents.push({ event: operation }); + const providerSpan = findOpenAISpan( + events, + operation.span.id, + spec.childNames, + )!; relevantEvents.push({ - event: findOpenAISpan(events, operation.span.id, spec.childNames)!, + event: providerSpan, summaryName: spec.childNames[0], }); + if (spec.nestedChildNames) { + relevantEvents.push( + ...findOpenAISpans(events, providerSpan.span.id, spec.nestedChildNames) + .sort((left, right) => + batchPrompt(left).localeCompare(batchPrompt(right)), + ) + .map((event) => ({ + event, + summaryName: spec.nestedChildNames?.[0], + })), + ); + } } return relevantEvents; @@ -599,22 +420,13 @@ function buildSpanTree( ): SpanTreeEntry[] { return buildRelevantEvents(events, operationSpecs).map( ({ event, summaryName }) => { - const summary = summarizeOpenAIPayload(event, summaryName) as Record< - string, - Json - >; - const { name: _name, type: _type, ...fields } = summary; - return { event, fields: { - span_attributes: spanTreeFields(event).span_attributes, - ...fields, + ...spanTreeFields(event), + context: event.context, }, - name: - typeof summary.name === "string" - ? summary.name - : (summaryName ?? event.span.name), + name: summaryName ?? event.span.name, }; }, ); @@ -688,6 +500,17 @@ export function defineOpenAIInstrumentationAssertions(options: { spec.childNames, ); expect(spanInstrumentationName(span)).toBe("openai"); + if (spec.nestedChildNames) { + const nested = findOpenAISpans( + events, + span?.span.id, + spec.nestedChildNames, + ); + expect(nested).toHaveLength(EXPECTED_BATCH_OUTPUTS.size); + for (const child of nested) { + expect(spanInstrumentationName(child)).toBe("openai"); + } + } } }); @@ -724,9 +547,12 @@ export function defineOpenAIInstrumentationAssertions(options: { expect(span?.row.metadata).toMatchObject({ provider: "openai", }); - expect( - typeof (span?.row.metadata as { model?: unknown } | undefined)?.model, - ).toBe("string"); + if (spec.expectsModel !== false) { + expect( + typeof (span?.row.metadata as { model?: unknown } | undefined) + ?.model, + ).toBe("string"); + } if (spec.expectsOutput) { expect(span?.output).toBeDefined(); @@ -744,6 +570,41 @@ export function defineOpenAIInstrumentationAssertions(options: { ); } + if (spec.nestedChildNames) { + const nested = findOpenAISpans( + events, + span?.span.id, + spec.nestedChildNames, + ); + expect(nested).toHaveLength(EXPECTED_BATCH_OUTPUTS.size); + expect(span?.span.parentIds).toEqual([operation?.span.id ?? ""]); + expect(nested.map(batchPrompt).sort()).toEqual( + [...EXPECTED_BATCH_OUTPUTS.keys()].sort(), + ); + for (const child of nested) { + const prompt = batchPrompt(child); + const firstChoice = Array.isArray(child.output) + ? asRecord(child.output[0]) + : undefined; + const message = asRecord(firstChoice?.message); + + expect(child.span.parentIds).toEqual([span?.span.id ?? ""]); + expect(child.row.metadata).toMatchObject({ + model: expect.any(String), + provider: "openai", + }); + const expectedOutput = EXPECTED_BATCH_OUTPUTS.get(prompt); + if (expectedOutput) { + expect(message?.content).toBe(expectedOutput); + expect(child.row.error).toBeUndefined(); + } else { + expect(child.output).toBeUndefined(); + expect(child.row.error).toContain("Batch fixture request failed"); + } + expect(child.metrics?.time_to_first_token).toBeUndefined(); + } + } + spec.validate?.(span); }); } diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index 076c39492..48199411e 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -7,6 +7,11 @@ import { runOperation, runTracedScenario, } from "../../helpers/provider-runtime.mjs"; +import { + completeOpenAIBatchTrace, + openaiBatchesRetrieveTraced, + openaiFilesCreateTraced, +} from "braintrust"; const OPENAI_MODEL = "gpt-4o-mini-2024-07-18"; const EMBEDDING_MODEL = "text-embedding-3-small"; @@ -104,6 +109,87 @@ function createMockStreamingClient(options) { : baseClient; } +function createMockBatchClient(options) { + const batchCreatedAt = Date.now() / 1000; + const baseClient = new options.OpenAI({ + apiKey: process.env.OPENAI_API_KEY ?? "test-openai-key", + baseURL: "https://example.test/v1", + fetch: async (url, init) => { + const pathname = new URL(String(url)).pathname; + if (pathname.endsWith("/files")) { + return new Response( + JSON.stringify({ + id: "file_batch_e2e_fixture", + object: "file", + bytes: 1024, + created_at: batchCreatedAt, + filename: "batch.jsonl", + purpose: "batch", + status: "processed", + }), + { + headers: { + "content-type": "application/json", + "x-request-id": "req_file_batch_e2e_fixture", + }, + status: 200, + }, + ); + } + + if (pathname.endsWith("/batches/batch_e2e_fixture")) { + return new Response( + JSON.stringify({ + id: "batch_e2e_fixture", + object: "batch", + endpoint: "/v1/chat/completions", + input_file_id: "file_batch_e2e_fixture", + completion_window: "24h", + status: "completed", + created_at: batchCreatedAt, + in_progress_at: batchCreatedAt + 1, + completed_at: batchCreatedAt + 2, + request_counts: { completed: 2, failed: 1, total: 3 }, + }), + { + headers: { + "content-type": "application/json", + "x-request-id": "req_batch_e2e_fixture", + }, + status: 200, + }, + ); + } + + const params = JSON.parse(String(init?.body)); + return new Response( + JSON.stringify({ + id: "batch_e2e_fixture", + object: "batch", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + completion_window: params.completion_window, + status: "validating", + created_at: 1_740_000_000, + metadata: params.metadata, + request_counts: { completed: 0, failed: 0, total: 0 }, + }), + { + headers: { + "content-type": "application/json", + "x-request-id": "req_batch_e2e_fixture", + }, + status: 200, + }, + ); + }, + }); + + return options.decorateClient + ? options.decorateClient(baseClient) + : baseClient; +} + export async function runOpenAIInstrumentationScenario(options) { const baseClient = new options.OpenAI({ apiKey: process.env.OPENAI_API_KEY, @@ -113,6 +199,7 @@ export async function runOpenAIInstrumentationScenario(options) { ? options.decorateClient(baseClient) : baseClient; const streamFixtureClient = createMockStreamingClient(options); + const batchFixtureClient = createMockBatchClient(options); const openAIMajorVersion = parseMajorVersion(options.openaiSdkVersion); const shouldCheckPrivateFieldMethods = typeof options.decorateClient === "function" && @@ -582,6 +669,107 @@ export async function runOpenAIInstrumentationScenario(options) { }, ); } + + await runOperation("openai-batch-operation", "batch", async () => { + const batchItems = [ + { + customId: "batch_chat_alpha", + prompt: "Reply with exactly ALPHA.", + response: "ALPHA", + }, + { + customId: "batch_chat_bravo", + prompt: "Reply with exactly BRAVO.", + response: "BRAVO", + }, + { + customId: "batch_chat_charlie", + prompt: "Reply with exactly CHARLIE.", + response: "CHARLIE", + }, + ]; + const input = batchItems + .map((item) => + JSON.stringify({ + custom_id: item.customId, + method: "POST", + url: "/v1/chat/completions", + body: { + model: OPENAI_MODEL, + messages: [{ role: "user", content: item.prompt }], + }, + }), + ) + .join("\n"); + const inputFile = await openaiFilesCreateTraced( + batchFixtureClient.files, + )({ + file: new File([input], "batch.jsonl"), + purpose: "batch", + }); + const created = await batchFixtureClient.batches.create({ + input_file_id: inputFile.id, + completion_window: "24h", + endpoint: "/v1/chat/completions", + }); + const completed = await openaiBatchesRetrieveTraced( + batchFixtureClient.batches, + )(created.id); + // Start both content requests before awaiting either one. These promises + // model the APIPromise values returned by files.content(). + const outputFile = Promise.resolve( + new Response( + [batchItems[1], batchItems[0]] + .map((item, index) => + JSON.stringify({ + custom_id: item.customId, + response: { + status_code: 200, + body: { + choices: [ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: item.response, + }, + }, + ], + usage: { + prompt_tokens: 8 + index, + completion_tokens: 1, + total_tokens: 9 + index, + }, + }, + }, + }), + ) + .join("\n"), + ), + ); + const errorFile = Promise.resolve( + new Response( + JSON.stringify({ + custom_id: batchItems[2].customId, + error: { + code: "fixture_error", + message: "Batch fixture request failed", + }, + }), + ), + ); + await completeOpenAIBatchTrace({ + inputFileId: completed.input_file_id, + inputFileContent: input, + // Batch results are not guaranteed to preserve input order. + outputFileContent: outputFile, + errorFileContent: errorFile, + }); + if (!(await outputFile).bodyUsed || !(await errorFile).bodyUsed) { + throw new Error("Expected batch result responses to be consumed"); + } + }); }, metadata: { openaiSdkVersion: options.openaiSdkVersion, diff --git a/generated_types.json b/generated_types.json index 3d9798f92..2303b7183 100644 --- a/generated_types.json +++ b/generated_types.json @@ -4208,7 +4208,7 @@ "type": "null" } ], - "description": "For prompt-backed scorers: the preprocessor function to use for trace template variables. Set to null to disable preprocessing. If omitted, the traced project's default preprocessor will be used, falling back to the global 'thread' preprocessor." + "description": "For prompt-backed functions: the preprocessor function to use for trace template variables. Set to null to disable preprocessing. If omitted, the traced project's default preprocessor will be used, falling back to the global 'thread' preprocessor." }, "Project": { "type": "object", @@ -6057,6 +6057,17 @@ "required": ["data"], "description": "Dataset rows", "title": "dataset_rows" + }, + { + "type": "object", + "properties": { + "experiment_name": { + "type": "string" + } + }, + "required": ["experiment_name"], + "description": "Experiment whose inputs and outputs should be used as dataset inputs and expected values", + "title": "experiment_name" } ], "description": "The dataset to use" @@ -6525,7 +6536,7 @@ "type": "null" } ], - "description": "Execution scope for topic automation. Defaults to span-level execution." + "description": "Execution scope for topic automation." }, "data_scope": { "$ref": "#/components/schemas/TopicAutomationDataScope" @@ -6695,7 +6706,17 @@ }, "source_facet": { "type": "string", - "description": "The facet field name to use as input for classification" + "description": "Materialized facet field name used when source_facet_function is absent" + }, + "source_facet_function": { + "allOf": [ + { + "$ref": "#/components/schemas/SavedFunctionId" + }, + { + "description": "The stable function reference for the source facet" + } + ] }, "embedding_model": { "type": "string", diff --git a/js/dev/server.test.ts b/js/dev/server.test.ts new file mode 100644 index 000000000..2e1461a7e --- /dev/null +++ b/js/dev/server.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from "vitest"; +import { BraintrustState } from "../src/logger"; +import { _exportsForTestingOnly } from "./server"; +import { evalBodySchema } from "./types"; + +describe("remote eval data", () => { + test("uses a named experiment as the data source", async () => { + const request = evalBodySchema.parse({ + name: "remote-evaluator", + data: { experiment_name: "source-experiment" }, + }); + const state = new BraintrustState({}); + + await expect( + _exportsForTestingOnly.getDataset(state, request.data), + ).resolves.toEqual({ + _type: "BaseExperiment", + name: "source-experiment", + }); + }); +}); diff --git a/js/dev/server.ts b/js/dev/server.ts index ebce5c3f5..5db8b467f 100644 --- a/js/dev/server.ts +++ b/js/dev/server.ts @@ -1,6 +1,7 @@ import express, { NextFunction, Request, Response } from "express"; import cors from "cors"; import { + BaseExperiment, callEvaluatorData, Eval, EvalData, @@ -335,6 +336,8 @@ async function getDataset( environment: data.dataset_environment ?? undefined, _internal_btql: data._internal_btql ?? undefined, }); + } else if ("experiment_name" in data) { + return BaseExperiment({ name: data.experiment_name }); } else { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return data.data as EvalCase[]; @@ -398,3 +401,7 @@ function makeScorer( return ret; } + +export const _exportsForTestingOnly = { + getDataset, +}; diff --git a/js/src/exports.ts b/js/src/exports.ts index 2f4332842..91494e404 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -129,6 +129,12 @@ export { export { registerSandbox } from "./sandbox"; +export { + completeOpenAIBatchTrace, + openaiBatchesRetrieveTraced, + openaiFilesCreateTraced, +} from "./openai-batch"; + // Internal isomorph layer for platform-specific implementations import _internalIso from "./isomorph"; export { _internalIso }; diff --git a/js/src/generated_plain_types.ts b/js/src/generated_plain_types.ts index b03f1fddc..dcaf1935a 100644 --- a/js/src/generated_plain_types.ts +++ b/js/src/generated_plain_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 776b6c295e852681) -- do not modify +// Auto-generated file (content hash e392acbca346c32a) -- do not modify export type AclObjectTypeType = /** @@ -773,6 +773,26 @@ export type FacetPreprocessorIdType = code: string; } | null; +export type SavedFunctionIdType = + | { + /** + * @enum function + */ + type: "function"; + id: string; + version?: /** + * The version of the function + */ + string | undefined; + } + | { + /** + * @enum global + */ + type: "global"; + name: string; + function_type: FunctionTypeEnumType; + }; export type TopicMapGenerationSettingsType = { /** * @enum hdbscan, kmeans, community @@ -795,9 +815,15 @@ export type TopicMapDataType = { */ type: "topic_map"; /** - * The facet field name to use as input for classification + * Materialized facet field name used when source_facet_function is absent */ source_facet: string; + source_facet_function?: + | (SavedFunctionIdType & + /** + * The stable function reference for the source facet + */ unknown) + | undefined; /** * The embedding model to use for embedding facet values */ @@ -1517,26 +1543,6 @@ export type ObjectReferenceNullishType = * Indicates the event was copied from another object. */ | null; -export type SavedFunctionIdType = - | { - /** - * @enum function - */ - type: "function"; - id: string; - version?: /** - * The version of the function - */ - string | undefined; - } - | { - /** - * @enum global - */ - type: "global"; - name: string; - function_type: FunctionTypeEnumType; - }; export type DatasetEventType = { /** * A unique identifier for the dataset event. If you don't provide one, Braintrust will generate one for you @@ -2748,7 +2754,7 @@ export type PromptParserNullishType = { } | null; export type PreprocessorSavedFunctionIdType = /** - * For prompt-backed scorers: the preprocessor function to use for trace template variables. Set to null to disable preprocessing. If omitted, the traced project's default preprocessor will be used, falling back to the global 'thread' preprocessor. + * For prompt-backed functions: the preprocessor function to use for trace template variables. Set to null to disable preprocessing. If omitted, the traced project's default preprocessor will be used, falling back to the global 'thread' preprocessor. */ | { /** @@ -4946,7 +4952,7 @@ export type TopicAutomationConfigType = { topic_map_functions: Array; scope?: | /** - * Execution scope for topic automation. Defaults to span-level execution. + * Execution scope for topic automation. */ (SpanScopeType | TraceScopeType | GroupScopeType | null) | undefined; @@ -6393,6 +6399,12 @@ export type RunEvalType = { */ | { data: Array; + } + /** + * Experiment whose inputs and outputs should be used as dataset inputs and expected values + */ + | { + experiment_name: string; }; name?: /** * The name of the eval to run when multiple evals available diff --git a/js/src/generated_types.ts b/js/src/generated_types.ts index fd807dcc5..ebf0f4be8 100644 --- a/js/src/generated_types.ts +++ b/js/src/generated_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 776b6c295e852681) -- do not modify +// Auto-generated file (content hash e392acbca346c32a) -- do not modify import { z } from "zod/v3"; @@ -273,6 +273,19 @@ export const FacetPreprocessorId = z.union([ z.null(), ]); export type FacetPreprocessorIdType = z.infer; +export const SavedFunctionId = z.union([ + z.object({ + type: z.literal("function"), + id: z.string(), + version: z.string().optional(), + }), + z.object({ + type: z.literal("global"), + name: z.string(), + function_type: FunctionTypeEnum.optional().default("scorer"), + }), +]); +export type SavedFunctionIdType = z.infer; export const TopicMapGenerationSettings = z.object({ algorithm: z.enum(["hdbscan", "kmeans", "community"]), dimension_reduction: z.enum(["umap", "pca", "none"]), @@ -289,6 +302,7 @@ export type TopicMapGenerationSettingsType = z.infer< export const TopicMapData = z.object({ type: z.literal("topic_map"), source_facet: z.string(), + source_facet_function: SavedFunctionId.and(z.unknown()).optional(), embedding_model: z.string(), bundle_key: z.string().optional(), report_key: z.string().optional(), @@ -629,19 +643,6 @@ export const ObjectReferenceNullish = z.union([ z.null(), ]); export type ObjectReferenceNullishType = z.infer; -export const SavedFunctionId = z.union([ - z.object({ - type: z.literal("function"), - id: z.string(), - version: z.string().optional(), - }), - z.object({ - type: z.literal("global"), - name: z.string(), - function_type: FunctionTypeEnum.optional().default("scorer"), - }), -]); -export type SavedFunctionIdType = z.infer; export const DatasetEvent = z.object({ id: z.string(), _xact_id: z.string(), @@ -2170,6 +2171,7 @@ export const RunEval = z.object({ .optional(), }), z.object({ data: z.array(z.unknown()) }), + z.object({ experiment_name: z.string() }), ]), name: z.string().optional(), parameters: z.object({}).partial().passthrough().optional(), diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 4ed98d53e..071869f6a 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -406,9 +406,9 @@ export class BraintrustPlugin extends BasePlugin { } } -// Re-export utility functions from OpenAIPlugin for backward compatibility +// Re-export OpenAI instrumentation utilities from their canonical modules. export { parseMetricsFromUsage, - processImagesInOutput, aggregateChatCompletionChunks, } from "./plugins/openai-plugin"; +export { processImagesInOutput } from "./plugins/openai-span-data"; diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts new file mode 100644 index 000000000..96a2809ef --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts @@ -0,0 +1,397 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + _exportsForTestingOnly, + initLogger, + startSpan, + withCurrent, +} from "../../logger"; +import { configureNode } from "../../node/config"; +import { + completeOpenAIBatchTrace, + openaiBatchesRetrieveTraced, + openaiFilesCreateTraced, +} from "../../openai-batch"; +import type { + OpenAIAPIPromise, + OpenAIBatchLike, + OpenAIEnhancedResponse, +} from "../../openai-batch-types"; +import { SpanComponentsV4 } from "../../../util/span_identifier_v4"; +import { OpenAIPlugin } from "./openai-plugin"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +const chatInput = [ + { + custom_id: "ok", + method: "POST", + url: "/v1/chat/completions", + body: { + model: "gpt-test", + messages: [{ role: "user", content: "hi" }], + }, + }, + { + custom_id: "bad", + method: "POST", + url: "/v1/chat/completions", + body: { + model: "gpt-test", + messages: [{ role: "user", content: "fail" }], + }, + }, +]; + +function jsonl(records: unknown[]): string { + return records.map((record) => JSON.stringify(record)).join("\n"); +} + +function apiPromise(data: T): OpenAIAPIPromise { + return asyncAPIPromise(Promise.resolve(data)); +} + +function asyncAPIPromise(dataPromise: Promise): OpenAIAPIPromise { + const promise = dataPromise as OpenAIAPIPromise; + let enhancedResponse: Promise> | undefined; + promise.withResponse = () => { + enhancedResponse ??= dataPromise.then((data) => ({ + data, + response: new Response(JSON.stringify(data), { + headers: { "content-type": "application/json" }, + }), + request_id: "request-id", + })); + return enhancedResponse; + }; + promise.asResponse = async () => (await promise.withResponse()).response; + return promise; +} + +async function uploadedText(upload: unknown): Promise { + if (upload instanceof Blob) { + return await upload.text(); + } + const chunks: Uint8Array[] = []; + for await (const chunk of upload as AsyncIterable) { + chunks.push(chunk); + } + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +} + +describe("OpenAI Batch instrumentation", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + let plugin: OpenAIPlugin; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "openai-batch-instrumentation.test.ts", + projectId: "test-project-id", + }); + plugin = new OpenAIPlugin(); + plugin.enable(); + }); + + afterEach(() => { + plugin.disable(); + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + it("tees upload streams and starts spans keyed by file ID and custom_id", async () => { + const source = jsonl(chatInput); + let received = ""; + const files = { + marker: "files-resource", + create( + this: { marker: string }, + params: { file: unknown; purpose: string }, + ) { + expect(this.marker).toBe("files-resource"); + return asyncAPIPromise( + Promise.resolve({ id: "file_stream", purpose: params.purpose }).then( + async (file) => { + received = await uploadedText(params.file); + return file; + }, + ), + ); + }, + }; + const encoder = new TextEncoder(); + const upload = { + path: "/tmp/batch.jsonl", + async *[Symbol.asyncIterator]() { + yield encoder.encode(source.slice(0, 19)); + yield encoder.encode(source.slice(19)); + }, + }; + + const file = await openaiFilesCreateTraced(files)({ + file: upload, + purpose: "batch", + }); + + expect(file.id).toBe("file_stream"); + expect(received).toBe(source); + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(3); + expect( + rows.find((row) => row.span_attributes?.name === "openai.batch"), + ).toMatchObject({ + metadata: { + endpoint: "/v1/chat/completions", + input_file_id: "file_stream", + provider: "openai", + }, + span_attributes: { type: "task" }, + }); + const children = rows.filter( + (row) => row.span_attributes?.name === "Chat Completion", + ); + expect(children).toHaveLength(2); + expect(children.map((row) => row.metadata.custom_id).sort()).toEqual([ + "bad", + "ok", + ]); + expect(children.map((row) => row.input[0].content).sort()).toEqual([ + "fail", + "hi", + ]); + expect(children.every((row) => row.metrics.end === undefined)).toBe(true); + }); + + it("preserves OpenAI promise helpers and passes non-batch files through", async () => { + let calls = 0; + const files = { + create( + params: { file: Blob; purpose: string }, + options?: { timeout: number }, + ) { + calls++; + expect(options).toEqual({ timeout: 10 }); + return apiPromise({ id: `file_${params.purpose}` }); + }, + }; + const create = openaiFilesCreateTraced(files); + + const traced = create( + { file: new Blob([jsonl([chatInput[0]])]), purpose: "batch" }, + { timeout: 10 }, + ); + expect(typeof traced.withResponse).toBe("function"); + expect(typeof traced.asResponse).toBe("function"); + await expect(traced.withResponse()).resolves.toMatchObject({ + data: { id: "file_batch" }, + request_id: "request-id", + }); + + const ordinary = create( + { file: new Blob(["ordinary"]), purpose: "assistants" }, + { timeout: 10 }, + ); + await expect(ordinary).resolves.toEqual({ id: "file_assistants" }); + expect(calls).toBe(2); + expect(await backgroundLogger.drain()).toHaveLength(2); + }); + + it("uses stable span IDs when the same file is observed again", async () => { + let uploadCount = 0; + const files = { + create() { + uploadCount++; + return apiPromise({ id: "file_stable" }); + }, + }; + const create = openaiFilesCreateTraced(files); + const params = { + file: new Blob([jsonl([chatInput[0]])]), + purpose: "batch", + }; + + await create(params); + const first = (await backgroundLogger.drain()) as Array< + Record + >; + await create(params); + const second = (await backgroundLogger.drain()) as Array< + Record + >; + + expect(uploadCount).toBe(2); + expect(second.map((row) => row.id).sort()).toEqual( + first.map((row) => row.id).sort(), + ); + }); + + it("retrieves batches through the resource and corrects lifecycle timestamps", async () => { + const parent = startSpan({ name: "parent" }); + await withCurrent(parent, async () => { + await openaiFilesCreateTraced({ + create() { + return apiPromise({ id: "file_timestamps" }); + }, + })({ file: new Blob([jsonl(chatInput)]), purpose: "batch" }); + }); + await backgroundLogger.drain(); + const batch: OpenAIBatchLike = { + id: "batch_timestamps", + endpoint: "/v1/chat/completions", + input_file_id: "file_timestamps", + status: "completed", + created_at: 100, + in_progress_at: 110, + completed_at: 200, + }; + const batches = { + marker: "batches-resource", + retrieve(this: { marker: string }, batchId: string) { + expect(this.marker).toBe("batches-resource"); + expect(batchId).toBe("batch_timestamps"); + return apiPromise(batch); + }, + }; + + const result = await openaiBatchesRetrieveTraced(batches)(batch.id); + const parentComponents = SpanComponentsV4.fromStr(await parent.export()); + parent.end(); + + expect(result).toBe(batch); + const rows = (await backgroundLogger.drain()) as Array>; + const task = rows.find( + (row) => row.span_attributes?.name === "openai.batch", + ); + const children = rows.filter( + (row) => row.span_attributes?.name === "Chat Completion", + ); + expect(task?.metrics).toMatchObject({ start: 100, end: 200 }); + expect(task?.span_parents).toEqual([parentComponents.data.span_id]); + expect(task?.root_span_id).toBe(parentComponents.data.root_span_id); + expect(children).toHaveLength(2); + expect( + children.every( + (row) => row.metrics.start === 110 && row.metrics.end === 200, + ), + ).toBe(true); + }); + + it("completes result spans by custom_id and consumes supplied responses", async () => { + await openaiFilesCreateTraced({ + create() { + return apiPromise({ id: "file_results" }); + }, + })({ file: new Blob([jsonl(chatInput)]), purpose: "batch" }); + const pendingRows = (await backgroundLogger.drain()) as Array< + Record + >; + const pendingIds = pendingRows.map((row) => row.id).sort(); + await openaiBatchesRetrieveTraced({ + retrieve() { + return apiPromise({ + id: "batch_results", + endpoint: "/v1/chat/completions", + input_file_id: "file_results", + status: "completed", + created_at: 100, + in_progress_at: 110, + completed_at: 200, + }); + }, + })("batch_results"); + await backgroundLogger.drain(); + const inputFileContent = new Response(jsonl(chatInput)); + const outputFileContent = new Response( + jsonl([ + { + custom_id: "ok", + response: { + status_code: 200, + body: { + choices: [ + { + index: 0, + message: { role: "assistant", content: "hello" }, + }, + ], + model: "gpt-result", + usage: { + prompt_tokens: 2, + completion_tokens: 3, + total_tokens: 5, + }, + }, + }, + }, + ]), + ); + const errorFileContent = new Response( + jsonl([ + { + custom_id: "bad", + error: { code: "server_error", message: "failed" }, + }, + ]), + ); + + await completeOpenAIBatchTrace({ + inputFileId: "file_results", + inputFileContent, + outputFileContent, + errorFileContent, + }); + + expect(inputFileContent.bodyUsed).toBe(true); + expect(outputFileContent.bodyUsed).toBe(true); + expect(errorFileContent.bodyUsed).toBe(true); + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows.map((row) => row.id).sort()).toEqual(pendingIds); + expect(rows.find((row) => row.output !== undefined)).toMatchObject({ + metadata: { model: "gpt-result", provider: "openai" }, + metrics: { + completion_tokens: 3, + end: 200, + prompt_tokens: 2, + tokens: 5, + }, + output: [{ index: 0, message: { role: "assistant", content: "hello" } }], + }); + expect(rows.find((row) => row.error !== undefined)?.error).toContain( + "failed", + ); + }); + + it("does not interfere with uploads when the JSONL is invalid", async () => { + const content = "not jsonl"; + let received = ""; + const result = await openaiFilesCreateTraced({ + create(params: { file: Blob; purpose: string }) { + return asyncAPIPromise( + Promise.resolve({ id: "file_invalid" }).then(async (file) => { + received = await params.file.text(); + return file; + }), + ); + }, + })({ file: new Blob([content]), purpose: "batch" }); + + expect(result).toEqual({ id: "file_invalid" }); + expect(received).toBe(content); + expect(await backgroundLogger.drain()).toEqual([]); + }); +}); diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts new file mode 100644 index 000000000..560ac51ca --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -0,0 +1,723 @@ +import { debugLogger } from "../../debug-logger"; +import { + _internalStartSpanWithInitialMerge, + _internalStartSpanWithInitialMergeAndParentSpanIds, + getSpanParentObject, + NOOP_SPAN, + type Span, + withCurrent, +} from "../../logger"; +import { parseMetricsFromUsage } from "../../openai-utils"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { getCurrentUnixTimestamp } from "../../util"; +import { isObject, SpanTypeAttribute } from "../../../util/index"; +import { SpanComponentsV4 } from "../../../util/span_identifier_v4"; +import type { + CompleteOpenAIBatchTraceArgs, + OpenAIBatchFile, + OpenAIBatchLike, +} from "../../openai-batch-types"; +import { openAIChannels } from "./openai-channels"; +import { + extractOpenAIBatchInput, + processImagesInOutput, +} from "./openai-span-data"; + +const SUPPORTED_ENDPOINTS = new Set(["/v1/chat/completions", "/v1/responses"]); +const TERMINAL_STATUSES = new Set([ + "completed", + "failed", + "expired", + "cancelled", +]); + +type BatchInputRecord = { + customId: string; + spanData?: ReturnType; +}; + +type BatchInputs = { + endpoint?: string; + inputs: Map; + issues: Error[]; +}; + +type BatchTraceContext = { + inputFileId: string; + endpoint: string; + parent: string; + taskStartTime: number; + childStartTime: number; +}; + +type PendingBatchTrace = { + context: BatchTraceContext; + inputs: Map; + endTime?: number; + status?: string; +}; + +type BatchResultRecord = { + value: Record; + source: "output" | "error"; +}; + +const pendingBatchTraces = new Map(); + +function read(value: unknown, key: PropertyKey): unknown { + if (!isObject(value)) { + return undefined; + } + try { + return Reflect.get(value, key); + } catch { + return undefined; + } +} + +function isBatchRecordIterable( + value: unknown, +): value is Iterable | AsyncIterable { + if ( + value === null || + (typeof value !== "object" && typeof value !== "function") + ) { + return false; + } + return ( + typeof read(value, Symbol.iterator) === "function" || + typeof read(value, Symbol.asyncIterator) === "function" + ); +} + +function validCustomId(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 64; +} + +function logBatchInstrumentationError(context: string, error: unknown): void { + debugLogger.debug(`OpenAI Batch instrumentation ${context}:`, error); +} + +async function exportParent( + parent: { export(): Promise } | { toStr(): string }, +): Promise { + if ("toStr" in parent && typeof parent.toStr === "function") { + return parent.toStr(); + } + if ("export" in parent && typeof parent.export === "function") { + return await parent.export(); + } + return undefined; +} + +async function deterministicDigest(namespace: string, ...parts: string[]) { + const encoded = new TextEncoder().encode( + [namespace, ...parts].map((part) => `${part.length}:${part}`).join("\0"), + ); + return new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", encoded), + ); +} + +function digestHex(bytes: Uint8Array, length: number): string { + return Array.from(bytes.slice(0, length)) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function digestUuid(bytes: Uint8Array): string { + const hex = digestHex(bytes, 16); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +async function batchSpanIds(inputFileId: string): Promise<{ + rowId: string; + spanId: string; + rootSpanId: string; +}> { + const [row, span, root] = await Promise.all([ + deterministicDigest("openai:batch:row", inputFileId), + deterministicDigest("openai:batch:span", inputFileId), + deterministicDigest("openai:batch:root", inputFileId), + ]); + return { + rowId: digestUuid(row), + spanId: digestHex(span, 8), + rootSpanId: digestHex(root, 16), + }; +} + +async function childSpanIds(inputFileId: string, customId: string) { + const [row, span] = await Promise.all([ + deterministicDigest("openai:batch:child:row", inputFileId, customId), + deterministicDigest("openai:batch:child:span", inputFileId, customId), + ]); + return { rowId: digestUuid(row), spanId: digestHex(span, 8) }; +} + +async function startBatchSpan(context: BatchTraceContext): Promise { + const ids = await batchSpanIds(context.inputFileId); + const parent = SpanComponentsV4.fromStr(context.parent); + const hasParentSpan = Boolean( + parent.data.row_id && parent.data.span_id && parent.data.root_span_id, + ); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMergeAndParentSpanIds( + withSpanInstrumentationName( + { + name: "openai.batch", + type: SpanTypeAttribute.TASK, + parent: context.parent, + ...(!hasParentSpan + ? { + parentSpanIds: { + parentSpanIds: [], + rootSpanId: ids.rootSpanId, + }, + } + : {}), + spanId: ids.spanId, + startTime: context.taskStartTime, + event: { + id: ids.rowId, + metadata: { + endpoint: context.endpoint, + input_file_id: context.inputFileId, + provider: "openai", + }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function startBatchChild( + context: BatchTraceContext, + taskParent: string, + input: BatchInputRecord, +): Promise { + const ids = await childSpanIds(context.inputFileId, input.customId); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { + name: + context.endpoint === "/v1/chat/completions" + ? "Chat Completion" + : "openai.responses.create", + type: SpanTypeAttribute.LLM, + parent: taskParent, + spanId: ids.spanId, + startTime: context.childStartTime, + event: { + id: ids.rowId, + ...(input.spanData?.input !== undefined + ? { input: input.spanData.input } + : {}), + metadata: { + ...input.spanData?.metadata, + custom_id: input.customId, + provider: "openai", + }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function* jsonlRecords( + file: OpenAIBatchFile, + onIssue: (error: Error) => void = () => {}, +): AsyncGenerator { + const resolvedFile = await file; + if (typeof resolvedFile === "string") { + for (const line of resolvedFile.split("\n")) { + if (!line.trim()) { + continue; + } + try { + yield JSON.parse(line.replace(/\r$/, "")); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue(new Error("OpenAI Batch file contains malformed JSONL")); + } + } + return; + } + + const body = read(resolvedFile, "body"); + const getReader = read(body, "getReader"); + if (isObject(body) && typeof getReader === "function") { + let reader: unknown; + try { + reader = Reflect.apply(getReader, body, []); + const decoder = new TextDecoder(); + let pending = ""; + while (true) { + const readChunk = read(reader, "read"); + if (typeof readChunk !== "function") { + throw new Error("Response body stream has no read method"); + } + const chunk = await Reflect.apply(readChunk, reader, []); + if (!isObject(chunk)) { + throw new Error("Response body stream returned an invalid chunk"); + } + if (chunk.done === true) { + pending += decoder.decode(); + break; + } + if (!(chunk.value instanceof Uint8Array)) { + throw new Error("Response body stream returned a non-byte chunk"); + } + pending += decoder.decode(chunk.value, { stream: true }); + let newline = pending.indexOf("\n"); + while (newline !== -1) { + const line = pending.slice(0, newline).replace(/\r$/, ""); + pending = pending.slice(newline + 1); + if (line.trim()) { + try { + yield JSON.parse(line); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue(new Error("OpenAI Batch file contains malformed JSONL")); + } + } + newline = pending.indexOf("\n"); + } + } + if (pending.trim()) { + try { + yield JSON.parse(pending.replace(/\r$/, "")); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue(new Error("OpenAI Batch file contains malformed JSONL")); + } + } + } catch (error) { + logBatchInstrumentationError("could not read JSONL response body", error); + onIssue(new Error("OpenAI Batch response body could not be read")); + } finally { + const releaseLock = read(reader, "releaseLock"); + if (typeof releaseLock === "function") { + try { + Reflect.apply(releaseLock, reader, []); + } catch (error) { + logBatchInstrumentationError( + "could not release stream reader", + error, + ); + } + } + } + return; + } + + if (isBatchRecordIterable(resolvedFile)) { + for await (const record of resolvedFile) { + yield record; + } + return; + } + + logBatchInstrumentationError("skipped invalid JSONL source", resolvedFile); + onIssue(new Error("OpenAI Batch file source is invalid")); +} + +async function readBatchInputs(file: OpenAIBatchFile): Promise { + const inputs = new Map(); + const issues: Error[] = []; + let endpoint: string | undefined; + try { + for await (const value of jsonlRecords(file, (issue) => + issues.push(issue), + )) { + const customId = read(value, "custom_id"); + const url = read(value, "url"); + const body = read(value, "body"); + if ( + !validCustomId(customId) || + inputs.has(customId) || + read(value, "method") !== "POST" || + typeof url !== "string" || + !SUPPORTED_ENDPOINTS.has(url) || + (endpoint !== undefined && endpoint !== url) || + !isObject(body) + ) { + issues.push(new Error("OpenAI Batch input contains an invalid record")); + continue; + } + endpoint = url; + let spanData: BatchInputRecord["spanData"]; + try { + spanData = extractOpenAIBatchInput(url, body); + } catch (error) { + logBatchInstrumentationError("could not extract batch input", error); + } + inputs.set(customId, { customId, spanData }); + } + } catch (error) { + logBatchInstrumentationError("could not process input file", error); + issues.push(new Error("OpenAI Batch input file could not be processed")); + } + if (!endpoint || inputs.size === 0) { + issues.push(new Error("OpenAI Batch input contains no supported records")); + } + return { endpoint, inputs, issues }; +} + +async function writePendingSpans( + trace: PendingBatchTrace, + endTime?: number, +): Promise { + const task = await startBatchSpan(trace.context); + const taskParent = await task.export(); + for (const input of trace.inputs.values()) { + try { + const child = await startBatchChild(trace.context, taskParent, input); + if (endTime !== undefined) { + child.end({ endTime }); + } + } catch (error) { + logBatchInstrumentationError("could not write batch request span", error); + } + } + if (endTime !== undefined) { + if (trace.status && trace.status !== "completed") { + task.log({ error: new Error(`OpenAI Batch ${trace.status}`) }); + } + task.end({ endTime }); + } +} + +export const interceptOpenAIFilesCreateTraced: Parameters< + typeof openAIChannels.filesCreateTraced.intercept +>[0] = async (target, thisArg, args) => { + const startTime = getCurrentUnixTimestamp(); + const inputPromise = readBatchInputs(args[0].inputFileContent); + const parentPromise = exportParent(args[0].parent); + const file = await Reflect.apply(target, thisArg, args); + try { + const inputFileId = read(file, "id"); + const [inputData, exportedParent] = await Promise.all([ + inputPromise, + parentPromise, + ]); + if ( + typeof inputFileId !== "string" || + !exportedParent || + !inputData.endpoint || + inputData.issues.length > 0 + ) { + if (inputData.issues[0]) { + logBatchInstrumentationError( + "skipped invalid input file", + inputData.issues[0], + ); + } + return file; + } + const trace: PendingBatchTrace = { + context: { + inputFileId, + endpoint: inputData.endpoint, + parent: exportedParent, + taskStartTime: startTime, + childStartTime: startTime, + }, + inputs: inputData.inputs, + }; + pendingBatchTraces.set(inputFileId, trace); + await writePendingSpans(trace); + } catch (error) { + logBatchInstrumentationError("could not start batch spans", error); + } + return file; +}; + +function validTimestamp(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function terminalEndTime(batch: OpenAIBatchLike, startTime: number): number { + let timestamp: unknown; + switch (batch.status) { + case "completed": + timestamp = batch.completed_at; + break; + case "failed": + timestamp = batch.failed_at; + break; + case "expired": + timestamp = batch.expired_at; + break; + default: + timestamp = batch.cancelled_at; + } + return validTimestamp(timestamp) + ? Math.max(timestamp, startTime) + : Math.max(getCurrentUnixTimestamp(), startTime); +} + +async function updateBatchTimestamps(batch: OpenAIBatchLike): Promise { + const trace = pendingBatchTraces.get(batch.input_file_id); + if (!trace || trace.context.endpoint !== batch.endpoint) { + return; + } + if (validTimestamp(batch.created_at)) { + trace.context.taskStartTime = batch.created_at; + } + trace.context.childStartTime = validTimestamp(batch.in_progress_at) + ? Math.max(batch.in_progress_at, trace.context.taskStartTime) + : trace.context.taskStartTime; + trace.status = batch.status; + if (TERMINAL_STATUSES.has(batch.status)) { + trace.endTime = terminalEndTime(batch, trace.context.childStartTime); + } + await writePendingSpans(trace, trace.endTime); +} + +export const interceptOpenAIBatchesRetrieveTraced: Parameters< + typeof openAIChannels.batchesRetrieveTraced.intercept +>[0] = (target, thisArg, args) => + Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (batch) => { + try { + await updateBatchTimestamps(batch); + } catch (error) { + logBatchInstrumentationError("could not update batch timestamps", error); + } + return batch; + }); + +function errorFromResult(result: BatchResultRecord): Error | undefined { + const error = read(result.value, "error"); + if (isObject(error)) { + const message = read(error, "message"); + return new Error( + typeof message === "string" ? message : "OpenAI Batch request failed", + ); + } + const response = read(result.value, "response"); + const statusCode = read(response, "status_code"); + if ( + result.source === "error" || + (typeof statusCode === "number" && (statusCode < 200 || statusCode >= 300)) + ) { + const message = read(read(read(response, "body"), "error"), "message"); + return new Error( + typeof message === "string" ? message : "OpenAI Batch request failed", + ); + } + return undefined; +} + +async function completeBatchResult( + context: BatchTraceContext, + taskParent: string, + endTime: number, + input: BatchInputRecord, + result: BatchResultRecord, +): Promise { + const child = await startBatchChild(context, taskParent, input); + try { + const resultError = errorFromResult(result); + const responseBody = read(read(result.value, "response"), "body"); + if (resultError) { + child.log({ error: resultError }); + } else if (isObject(responseBody)) { + const model = read(responseBody, "model"); + child.log({ + output: + context.endpoint === "/v1/chat/completions" + ? read(responseBody, "choices") + : processImagesInOutput(read(responseBody, "output")), + ...(typeof model === "string" ? { metadata: { model } } : {}), + metrics: parseMetricsFromUsage(read(responseBody, "usage")), + }); + } else { + child.log({ error: new Error("OpenAI Batch response body is missing") }); + } + } catch (error) { + child.log({ error }); + } finally { + child.end({ endTime }); + } +} + +async function completeResultFile({ + context, + endTime, + file, + inputs, + issues, + seen, + source, + taskParent, +}: { + context: BatchTraceContext; + endTime: number; + file: OpenAIBatchFile | undefined; + inputs: Map; + issues: Error[]; + seen: Set; + source: BatchResultRecord["source"]; + taskParent: string; +}): Promise { + if (file === undefined) { + return; + } + for await (const value of jsonlRecords(file, (issue) => issues.push(issue))) { + const customId = read(value, "custom_id"); + if (!validCustomId(customId) || !isObject(value)) { + issues.push( + new Error("OpenAI Batch result is missing a valid custom_id"), + ); + continue; + } + if (seen.has(customId)) { + issues.push(new Error("OpenAI Batch result contains a duplicate")); + continue; + } + const input = inputs.get(customId); + if (!input) { + issues.push( + new Error("OpenAI Batch result does not match an input record"), + ); + continue; + } + seen.add(customId); + await completeBatchResult(context, taskParent, endTime, input, { + value, + source, + }); + } +} + +function sameInputs( + first: Map, + second: Map, +): boolean { + return ( + first.size === second.size && + [...first.keys()].every((customId) => second.has(customId)) + ); +} + +async function contextForCompletion( + inputFileId: string, + inputData: BatchInputs, +): Promise { + if (!inputData.endpoint || inputData.issues.length > 0) { + return undefined; + } + const existing = pendingBatchTraces.get(inputFileId); + if (existing) { + if ( + existing.context.endpoint !== inputData.endpoint || + !sameInputs(existing.inputs, inputData.inputs) + ) { + return undefined; + } + return { ...existing, inputs: inputData.inputs }; + } + const parent = await exportParent(getSpanParentObject()); + if (!parent) { + return undefined; + } + const startTime = getCurrentUnixTimestamp(); + return { + context: { + inputFileId, + endpoint: inputData.endpoint, + parent, + taskStartTime: startTime, + childStartTime: startTime, + }, + inputs: inputData.inputs, + }; +} + +async function completeBatch( + args: CompleteOpenAIBatchTraceArgs, +): Promise { + const inputData = await readBatchInputs(args.inputFileContent); + const trace = await contextForCompletion(args.inputFileId, inputData); + if (!trace) { + logBatchInstrumentationError( + "left batch spans pending", + inputData.issues[0] ?? new Error("OpenAI Batch input does not match"), + ); + return; + } + const endTime = trace.endTime ?? getCurrentUnixTimestamp(); + const task = await startBatchSpan(trace.context); + const taskParent = await task.export(); + const issues: Error[] = []; + const seen = new Set(); + await Promise.all([ + completeResultFile({ + context: trace.context, + endTime, + file: args.outputFileContent, + inputs: trace.inputs, + issues, + seen, + source: "output", + taskParent, + }), + completeResultFile({ + context: trace.context, + endTime, + file: args.errorFileContent, + inputs: trace.inputs, + issues, + seen, + source: "error", + taskParent, + }), + ]); + if (issues.length > 0) { + logBatchInstrumentationError("left batch spans pending", issues[0]); + return; + } + + const missing = [...trace.inputs.values()].filter( + ({ customId }) => !seen.has(customId), + ); + if (!trace.status || trace.status === "completed") { + if (missing.length > 0) { + logBatchInstrumentationError( + "left batch spans pending", + new Error("OpenAI Batch result files are incomplete"), + ); + return; + } + } else { + for (const input of missing) { + const child = await startBatchChild(trace.context, taskParent, input); + child.log({ error: new Error(`OpenAI Batch ${trace.status}`) }); + child.end({ endTime }); + } + task.log({ error: new Error(`OpenAI Batch ${trace.status}`) }); + } + task.end({ endTime }); + pendingBatchTraces.delete(args.inputFileId); +} + +export const interceptOpenAIBatchTraceComplete: Parameters< + typeof openAIChannels.batchesCompleteTrace.intercept +>[0] = (target, thisArg, args) => + Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (result) => { + try { + await completeBatch(args[0]); + } catch (error) { + logBatchInstrumentationError("could not complete batch", error); + } + return result; + }); diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index 4b1347609..5ef873eca 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -17,6 +17,13 @@ import type { OpenAIResponseCreateParams, OpenAIResponseStreamEvent, } from "../../vendor-sdk-types/openai"; +import type { + CompleteOpenAIBatchTraceArgs, + OpenAIBatchLike, + OpenAIBatchesRetrieveTraceArgs, + OpenAIFileLike, + OpenAIFilesCreateTraceArgs, +} from "../../openai-batch-types"; type OpenAIChatSpanInfo = NonNullable["span_info"]>; @@ -31,6 +38,33 @@ type OpenAIResponsesChannelExtras = OpenAIChannelExtras; export const openAIChannels = defineChannels( "openai", { + filesCreateTraced: channel< + [OpenAIFilesCreateTraceArgs], + OpenAIFileLike, + OpenAIChannelExtras + >({ + channelName: "files.create-traced", + kind: "async", + }), + + batchesRetrieveTraced: channel< + [OpenAIBatchesRetrieveTraceArgs], + OpenAIBatchLike, + OpenAIChannelExtras + >({ + channelName: "batches.retrieve-traced", + kind: "async", + }), + + batchesCompleteTrace: channel< + [CompleteOpenAIBatchTraceArgs], + void, + OpenAIChannelExtras + >({ + channelName: "batches.complete-trace", + kind: "async", + }), + chatCompletionsCreate: channel< [OpenAIChatCreateParams], OpenAIChatCompletion | OpenAIChatStream, diff --git a/js/src/instrumentation/plugins/openai-plugin.test.ts b/js/src/instrumentation/plugins/openai-plugin.test.ts index 5f2b237fa..b55e56d8a 100644 --- a/js/src/instrumentation/plugins/openai-plugin.test.ts +++ b/js/src/instrumentation/plugins/openai-plugin.test.ts @@ -1,9 +1,9 @@ import { describe, it, expect } from "vitest"; import { parseMetricsFromUsage, - processImagesInOutput, aggregateChatCompletionChunks, } from "./openai-plugin"; +import { processImagesInOutput } from "./openai-span-data"; import { Attachment } from "../../logger"; describe("parseMetricsFromUsage", () => { diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index d1ca89dcc..a534764ae 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -5,11 +5,20 @@ import { traceSyncStreamChannel, unsubscribeAll, } from "../core/channel-tracing"; -import { Attachment } from "../../logger"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; -import { processInputAttachments } from "../../wrappers/attachment-utils"; import { openAIChannels } from "./openai-channels"; +import { + extractOpenAIChatInput, + extractOpenAIResponsesInput, + extractOpenAIResponsesMetadata, + processImagesInOutput, +} from "./openai-span-data"; +import { + interceptOpenAIBatchesRetrieveTraced, + interceptOpenAIBatchTraceComplete, + interceptOpenAIFilesCreateTraced, +} from "./openai-batch-instrumentation"; import { BRAINTRUST_CACHED_STREAM_METRIC, getCachedMetricFromHeaders, @@ -38,18 +47,24 @@ export class OpenAIPlugin extends BasePlugin { } protected onEnable(): void { + this.unsubscribers.push( + openAIChannels.filesCreateTraced.intercept( + interceptOpenAIFilesCreateTraced, + ), + openAIChannels.batchesRetrieveTraced.intercept( + interceptOpenAIBatchesRetrieveTraced, + ), + openAIChannels.batchesCompleteTrace.intercept( + interceptOpenAIBatchTraceComplete, + ), + ); + // Chat Completions - supports streaming this.unsubscribers.push( traceStreamingChannel(openAIChannels.chatCompletionsCreate, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), extractOutput: (result) => { return result?.choices; }, @@ -101,13 +116,7 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.betaChatCompletionsParse, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), extractOutput: (result) => { return result?.choices; }, @@ -131,13 +140,7 @@ export class OpenAIPlugin extends BasePlugin { traceSyncStreamChannel(openAIChannels.betaChatCompletionsStream, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), }), ); @@ -171,23 +174,11 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.responsesCreate, { name: "openai.responses.create", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -208,13 +199,7 @@ export class OpenAIPlugin extends BasePlugin { traceSyncStreamChannel(openAIChannels.responsesStream, { name: "openai.responses.create", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractFromEvent: (event) => { if (event.type !== "response.completed" || !event.response) { return {}; @@ -243,23 +228,11 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.responsesParse, { name: "openai.responses.parse", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -280,23 +253,11 @@ export class OpenAIPlugin extends BasePlugin { traceAsyncChannel(openAIChannels.responsesCompact, { name: "openai.responses.compact", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -370,54 +331,6 @@ function withCachedMetric( }; } -/** - * Process output to convert base64 images to attachments. - * Used for Responses API image generation output. - */ -export function processImagesInOutput(output: any): any { - if (Array.isArray(output)) { - return output.map(processImagesInOutput); - } - - if (isObject(output)) { - if ( - output.type === "image_generation_call" && - output.result && - typeof output.result === "string" - ) { - const fileExtension = output.output_format || "png"; - const contentType = `image/${fileExtension}`; - - const baseFilename = - output.revised_prompt && typeof output.revised_prompt === "string" - ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") - : "generated_image"; - const filename = `${baseFilename}.${fileExtension}`; - - // Convert base64 string to Blob - const binaryString = atob(output.result); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const blob = new Blob([bytes], { type: contentType }); - - const attachment = new Attachment({ - data: blob, - filename: filename, - contentType: contentType, - }); - - return { - ...output, - result: attachment, - }; - } - } - - return output; -} - function mergeLogprobTokens( existing: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], incoming: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], diff --git a/js/src/instrumentation/plugins/openai-span-data.ts b/js/src/instrumentation/plugins/openai-span-data.ts new file mode 100644 index 000000000..2562315ad --- /dev/null +++ b/js/src/instrumentation/plugins/openai-span-data.ts @@ -0,0 +1,120 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Attachment } from "../../logger"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { isObject } from "../../../util/index"; + +const OPENAI_METADATA_KEYS = [ + "model", + "temperature", + "top_p", + "max_tokens", + "frequency_penalty", + "presence_penalty", + "stop", + "response_format", + "tools", + "tool_choice", + "parallel_tool_calls", + "max_tool_calls", +] as const; + +function batchMetadata(params: Record) { + const metadata: Record = { provider: "openai" }; + for (const key of OPENAI_METADATA_KEYS) { + try { + const value = Reflect.get(params, key); + if (value !== undefined) { + metadata[key] = value; + } + } catch { + // Ignore hostile getters in provider input. + } + } + return metadata; +} + +export function extractOpenAIBatchInput( + endpoint: string, + params: Record, +): { input: unknown; metadata: Record } { + const input = + endpoint === "/v1/chat/completions" ? params.messages : params.input; + return { + input: processInputAttachments(input), + metadata: batchMetadata(params), + }; +} + +export function extractOpenAIChatInput(params: Record): { + input: unknown; + metadata: Record; +} { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; +} + +export function extractOpenAIResponsesInput(params: Record): { + input: unknown; + metadata: Record; +} { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; +} + +export function extractOpenAIResponsesMetadata( + result: Record | undefined, +): Record | undefined { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +/** Convert Responses API base64 image outputs to Braintrust attachments. */ +export function processImagesInOutput(output: any): any { + if (Array.isArray(output)) { + return output.map(processImagesInOutput); + } + + if ( + isObject(output) && + output.type === "image_generation_call" && + typeof output.result === "string" && + output.result + ) { + const fileExtension = output.output_format || "png"; + const contentType = `image/${fileExtension}`; + const baseFilename = + typeof output.revised_prompt === "string" + ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") + : "generated_image"; + let binaryString: string; + try { + binaryString = atob(output.result); + } catch { + return output; + } + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + return { + ...output, + result: new Attachment({ + data: new Blob([bytes], { type: contentType }), + filename: `${baseFilename}.${fileExtension}`, + contentType, + }), + }; + } + + return output; +} diff --git a/js/src/logger.ts b/js/src/logger.ts index 9733c3030..4c545a833 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -6363,6 +6363,22 @@ export function _internalStartSpanWithInitialMerge< InitialSpanWriteAsMergeArg).span; } +/** @internal Start a deterministic span under an exported object-only parent. */ +export function _internalStartSpanWithInitialMergeAndParentSpanIds< + IsAsyncFlush extends boolean = true, +>(args: StartSpanArgs & AsyncFlushArg & OptionalStateArg): Span { + return startSpanAndIsLogger( + { + ...args, + [INITIAL_SPAN_WRITE_AS_MERGE]: true, + } as StartSpanArgs & + AsyncFlushArg & + OptionalStateArg & + InitialSpanWriteAsMergeArg, + { useParentSpanIdsForObjectParent: true }, + ).span; +} + /** @internal Start a span with SDK-controlled context fields. */ export function _internalStartSpanWithContext< IsAsyncFlush extends boolean = true, @@ -6399,6 +6415,7 @@ function startSpanAndIsLogger( AsyncFlushArg & OptionalStateArg & InternalSpanContextArg, + internalOptions?: { useParentSpanIdsForObjectParent?: boolean }, ): { span: Span; isSyncFlushLogger: boolean } { const state = args?.state ?? _globalState; @@ -6416,7 +6433,7 @@ function startSpanAndIsLogger( parentObject instanceof SpanComponentsV3 || parentObject instanceof SpanComponentsV4 ) { - const parentSpanIds: ParentSpanIds | undefined = + const parentSpanIds: ParentSpanIds | MultiParentSpanIds | undefined = parentObject.data.row_id && parentSpanIdsUsable( parentObject.data.span_id, @@ -6426,7 +6443,9 @@ function startSpanAndIsLogger( spanId: parentObject.data.span_id, rootSpanId: parentObject.data.root_span_id, } - : undefined; + : internalOptions?.useParentSpanIdsForObjectParent + ? args?.parentSpanIds + : undefined; // The parent object/state are already resolved from `parent` above; drop // the raw `parent` so it isn't re-normalized. const { parent: _ignoredParent, ...spanArgs } = args ?? {}; diff --git a/js/src/openai-batch-types.ts b/js/src/openai-batch-types.ts new file mode 100644 index 000000000..b6052d575 --- /dev/null +++ b/js/src/openai-batch-types.ts @@ -0,0 +1,70 @@ +export type OpenAIBatchJSONL = + | string + | Iterable + | AsyncIterable; + +export type OpenAIBatchFileContent = OpenAIBatchJSONL | Response; + +export type OpenAIBatchFile = + | OpenAIBatchFileContent + | PromiseLike; + +export interface OpenAIFileLike { + id: string; + [key: string]: unknown; +} + +export interface OpenAIBatchLike { + id: string; + endpoint: string; + input_file_id: string; + status: string; + created_at?: number | null; + in_progress_at?: number | null; + completed_at?: number | null; + failed_at?: number | null; + expired_at?: number | null; + cancelled_at?: number | null; + request_counts?: { + completed?: number; + failed?: number; + total?: number; + } | null; + [key: string]: unknown; +} + +export type OpenAIEnhancedResponse = { + response: Response; + data: T; + request_id?: string | null; +}; + +export interface OpenAIAPIPromise extends Promise { + withResponse(): Promise>; + asResponse(): Promise; +} + +export interface OpenAIFilesResource { + create(params: TParams, options?: TOptions): OpenAIAPIPromise; +} + +export interface OpenAIBatchesResource { + retrieve(batchId: string, options?: TOptions): OpenAIAPIPromise; +} + +export interface OpenAIFilesCreateTraceArgs { + inputFileContent: OpenAIBatchFileContent; + parent: { export(): Promise } | { toStr(): string }; + params: unknown; +} + +export interface OpenAIBatchesRetrieveTraceArgs { + batchId: string; +} + +export interface CompleteOpenAIBatchTraceArgs { + inputFileId: string; + inputFileContent: OpenAIBatchFile; + outputFileContent?: OpenAIBatchFile; + errorFileContent?: OpenAIBatchFile; +} diff --git a/js/src/openai-batch.ts b/js/src/openai-batch.ts new file mode 100644 index 000000000..fa0837684 --- /dev/null +++ b/js/src/openai-batch.ts @@ -0,0 +1,285 @@ +import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import { getSpanParentObject } from "./logger"; +import { + createLazyAPIPromise, + type EnhancedResponse, +} from "./wrappers/openai-promise-utils"; +import type { + CompleteOpenAIBatchTraceArgs, + OpenAIAPIPromise, + OpenAIBatchLike, + OpenAIBatchesResource, + OpenAIBatchFileContent, + OpenAIFileLike, + OpenAIFilesResource, +} from "./openai-batch-types"; + +function read(value: unknown, key: PropertyKey): unknown { + if ( + value === null || + (typeof value !== "object" && typeof value !== "function") + ) { + return undefined; + } + try { + return Reflect.get(value, key); + } catch { + return undefined; + } +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return typeof read(value, Symbol.asyncIterator) === "function"; +} + +function teeAsyncIterable(source: AsyncIterable): { + upload: AsyncIterable; + trace: Response; +} { + const iterator = source[Symbol.asyncIterator](); + const stream = new ReadableStream({ + async pull(controller) { + try { + const result = await iterator.next(); + if (result.done) { + controller.close(); + return; + } + if (!(result.value instanceof Uint8Array)) { + throw new TypeError( + "OpenAI upload streams must yield Uint8Array chunks", + ); + } + controller.enqueue(result.value); + } catch (error) { + controller.error(error); + } + }, + async cancel(reason) { + await iterator.return?.(reason); + }, + }); + const [uploadStream, trace] = stream.tee(); + const upload = { + async *[Symbol.asyncIterator]() { + const reader = uploadStream.getReader(); + try { + while (true) { + const result = await reader.read(); + if (result.done) { + return; + } + yield result.value; + } + } finally { + reader.releaseLock(); + } + }, + }; + const path = read(source, "path"); + if (typeof path === "string") { + Object.defineProperty(upload, "path", { value: path }); + } + return { upload, trace: new Response(trace) }; +} + +function teeUpload( + upload: unknown, +): { upload: unknown; trace: OpenAIBatchFileContent } | undefined { + if (upload instanceof Blob) { + return { upload, trace: new Response(upload) }; + } + if (upload instanceof Response) { + return { upload, trace: upload.clone() }; + } + if (isAsyncIterable(upload)) { + return teeAsyncIterable(upload); + } + return undefined; +} + +/** + * Wrap `openai.files.create` to trace an OpenAI Batch input file upload. + * + * Pass the `openai.files` resource, then call the returned function with the + * same arguments as `openai.files.create`. Files whose purpose is not `batch` + * are passed through without tracing. Blob and Response uploads are copied for + * tracing; async-iterable uploads (including Node file streams) are teed so the + * exact bytes still flow to OpenAI while Braintrust reads a separate branch. + * + * The returned input file ID identifies the trace. Each request span is + * identified by that file ID and its JSONL `custom_id`, so use every traced + * input file for only one OpenAI Batch. + * + * @example + * ```ts + * const inputFile = await openaiFilesCreateTraced(openai.files)({ + * file: createReadStream("batch.jsonl"), + * purpose: "batch", + * }); + * ``` + */ +export function openaiFilesCreateTraced< + TParams, + TFile extends OpenAIFileLike, + TOptions = unknown, +>( + files: OpenAIFilesResource, +): (params: TParams, options?: TOptions) => OpenAIAPIPromise { + return (params, options) => { + const parent = getSpanParentObject(); + const purpose = read(params, "purpose"); + const file = read(params, "file"); + if (purpose !== "batch") { + return files.create(params, options); + } + + let branches: ReturnType; + try { + branches = teeUpload(file); + } catch { + return files.create(params, options); + } + if (!branches) { + return files.create(params, options); + } + + const tracedParams = { + ...Object(params), + file: branches.upload, + } as TParams; + const apiPromise = files.create(tracedParams, options); + let enhancedResponse: EnhancedResponse | undefined; + const tracedResponse = openAIChannels.filesCreateTraced + .invoke( + async () => { + enhancedResponse = await apiPromise.withResponse(); + return enhancedResponse.data; + }, + files, + [{ params, inputFileContent: branches.trace, parent }], + {}, + ) + .then((data) => { + if (!enhancedResponse) { + throw new Error( + "Expected OpenAI withResponse() to provide a response", + ); + } + return { ...enhancedResponse, data }; + }); + + return createLazyAPIPromise( + () => tracedResponse, + () => tracedResponse.then(({ response }) => response), + () => apiPromise, + ); + }; +} + +/** + * Wrap `openai.batches.retrieve` to update an OpenAI Batch trace with the + * server-recorded lifecycle timestamps and close its spans at a terminal + * status. + * + * Pass the `openai.batches` resource, then call the returned function with the + * same arguments as `openai.batches.retrieve`. The Batch must use an input file + * previously uploaded through `openaiFilesCreateTraced` in this process. + * + * @example + * ```ts + * const batch = await openaiBatchesRetrieveTraced(openai.batches)(batchId); + * ``` + */ +export function openaiBatchesRetrieveTraced< + TBatch extends OpenAIBatchLike, + TOptions = unknown, +>( + batches: OpenAIBatchesResource, +): (batchId: string, options?: TOptions) => OpenAIAPIPromise { + return (batchId, options) => { + const apiPromise = batches.retrieve(batchId, options); + let enhancedResponse: EnhancedResponse | undefined; + const tracedResponse = openAIChannels.batchesRetrieveTraced + .invoke( + async () => { + enhancedResponse = await apiPromise.withResponse(); + return enhancedResponse.data; + }, + batches, + [{ batchId }], + {}, + ) + .then((data) => { + if (!enhancedResponse) { + throw new Error( + "Expected OpenAI withResponse() to provide a response", + ); + } + return { ...enhancedResponse, data }; + }); + + return createLazyAPIPromise( + () => tracedResponse, + () => tracedResponse.then(({ response }) => response), + () => apiPromise, + ); + }; +} + +/** + * Add result data to spans created by `openaiFilesCreateTraced`. + * + * This helper performs no OpenAI API requests. Pass the input file ID and the + * input, output, and error contents returned by `openai.files.content`, or pass + * JSONL strings/iterables directly. Response bodies are consumed. Call + * `openaiBatchesRetrieveTraced` first when exact OpenAI lifecycle timestamps + * are available. + * + * @example + * ```ts + * const outputFileContent = batch.output_file_id + * ? await openai.files.content(batch.output_file_id) + * : undefined; + * const errorFileContent = batch.error_file_id + * ? await openai.files.content(batch.error_file_id) + * : undefined; + * await completeOpenAIBatchTrace({ + * inputFileId: batch.input_file_id, + * inputFileContent: await openai.files.content(batch.input_file_id), + * outputFileContent, + * errorFileContent, + * }); + * ``` + */ +export async function completeOpenAIBatchTrace( + args: CompleteOpenAIBatchTraceArgs, +): Promise { + const inputFileContent = Promise.resolve(args.inputFileContent); + const outputFileContent = + args.outputFileContent === undefined + ? undefined + : Promise.resolve(args.outputFileContent); + const errorFileContent = + args.errorFileContent === undefined + ? undefined + : Promise.resolve(args.errorFileContent); + + void inputFileContent.catch(() => undefined); + void outputFileContent?.catch(() => undefined); + void errorFileContent?.catch(() => undefined); + + await openAIChannels.batchesCompleteTrace.invoke( + async () => undefined, + undefined, + [ + { + ...args, + inputFileContent, + outputFileContent, + errorFileContent, + }, + ], + {}, + ); +} diff --git a/js/src/openai-promise-utils.test.ts b/js/src/openai-promise-utils.test.ts new file mode 100644 index 000000000..dc474c316 --- /dev/null +++ b/js/src/openai-promise-utils.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import { wrapOpenAI } from "./wrappers/oai"; +import { + type APIPromise, + createLazyAPIPromise, +} from "./wrappers/openai-promise-utils"; + +function apiPromise(data: T, requestId: string): APIPromise { + const response = new Response(JSON.stringify(data), { + headers: { "content-type": "application/json" }, + }); + const promise = Promise.resolve(data) as APIPromise; + promise.withResponse = async () => ({ + data, + response, + request_id: requestId, + }); + promise.asResponse = async () => response; + return promise; +} + +describe("OpenAI API promise wrappers", () => { + it("forwards asResponse without parsing the response body", async () => { + const response = new Response(JSON.stringify({ value: "raw" }), { + headers: { "content-type": "application/json" }, + }); + const nativePromise = apiPromise({ value: "parsed" }, "req_raw"); + let tracedWithResponseCalls = 0; + let tracedAsResponseCalls = 0; + const wrapped = createLazyAPIPromise( + async () => { + tracedWithResponseCalls++; + return { data: { value: "parsed" }, response }; + }, + async () => { + tracedAsResponseCalls++; + return response; + }, + () => nativePromise, + ); + + const raw = await wrapped.asResponse(); + expect(tracedWithResponseCalls).toBe(0); + expect(tracedAsResponseCalls).toBe(1); + await expect(raw.json()).resolves.toEqual({ value: "raw" }); + }); + + it("does not trace the request again when parsed after asResponse", async () => { + const response = new Response(JSON.stringify({ value: "raw" }), { + headers: { "content-type": "application/json" }, + }); + const nativePromise = apiPromise({ value: "parsed" }, "req_raw"); + let tracedWithResponseCalls = 0; + let tracedAsResponseCalls = 0; + const wrapped = createLazyAPIPromise( + async () => { + tracedWithResponseCalls++; + return { data: { value: "parsed" }, response }; + }, + async () => { + tracedAsResponseCalls++; + return response; + }, + () => nativePromise, + ); + + await wrapped.asResponse(); + await expect(wrapped).resolves.toEqual({ value: "parsed" }); + expect(tracedWithResponseCalls).toBe(0); + expect(tracedAsResponseCalls).toBe(1); + }); + + it("dispatches the OpenAI channel lifecycle for asResponse-only calls", async () => { + const phases: string[] = []; + let result: unknown; + const tracingChannel = + openAIChannels.chatCompletionsCreate.tracingChannel(); + const handlers = { + asyncEnd: (event: { result?: unknown }) => { + phases.push("asyncEnd"); + result = event.result; + }, + error: () => phases.push("error"), + start: () => phases.push("start"), + }; + tracingChannel.subscribe(handlers); + + try { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown }) => + apiPromise({ choices: [] }, "req_raw"), + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + const response = await client.chat.completions + .create({ messages: [] }) + .asResponse(); + expect(response.bodyUsed).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 0)); + } finally { + tracingChannel.unsubscribe(handlers); + } + + expect(phases).toEqual(["start", "asyncEnd"]); + expect(result).toBeUndefined(); + }); + + it("does not consume a raw response body and preserves cancellation", async () => { + let cancelled = false; + let pulls = 0; + const response = new Response( + new ReadableStream( + { + pull(controller) { + pulls++; + controller.enqueue( + new TextEncoder().encode(JSON.stringify({ choices: [] })), + ); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ), + { headers: { "content-type": "application/json" } }, + ); + const promise: APIPromise<{ choices: unknown[] }> = Object.assign( + Promise.resolve({ choices: [] as unknown[] }), + { + asResponse: async () => response, + withResponse: async () => ({ data: { choices: [] }, response }), + }, + ); + const phases: string[] = []; + const tracingChannel = + openAIChannels.chatCompletionsCreate.tracingChannel(); + const handlers = { + asyncEnd: () => phases.push("asyncEnd"), + error: () => phases.push("error"), + start: () => phases.push("start"), + }; + tracingChannel.subscribe(handlers); + + try { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown }) => promise, + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + const raw = await client.chat.completions + .create({ messages: [] }) + .asResponse(); + expect(raw).toBe(response); + expect(raw.bodyUsed).toBe(false); + expect(pulls).toBe(0); + expect(phases).toEqual(["start", "asyncEnd"]); + + const reader = raw.body?.getReader(); + await reader?.read(); + await reader?.cancel(); + expect(cancelled).toBe(true); + } finally { + tracingChannel.unsubscribe(handlers); + } + }); + + it.each([false, true])( + "preserves request_id for chat responses with stream=%s", + async (stream) => { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown; stream?: boolean }) => + apiPromise({ choices: [] }, "req_chat"), + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + await expect( + client.chat.completions.create({ messages: [], stream }).withResponse(), + ).resolves.toMatchObject({ request_id: "req_chat" }); + }, + ); +}); diff --git a/js/src/vendor-sdk-types/openai-common.ts b/js/src/vendor-sdk-types/openai-common.ts index d2dca2323..15f88a3c9 100644 --- a/js/src/vendor-sdk-types/openai-common.ts +++ b/js/src/vendor-sdk-types/openai-common.ts @@ -8,11 +8,13 @@ export interface OpenAIAPIPromise extends Promise { withResponse(): Promise>; + asResponse(): Promise; } export interface OpenAIWithResponse { data: T; response: Response; + request_id?: string | null; } // Requests diff --git a/js/src/wrappers/oai.ts b/js/src/wrappers/oai.ts index b2fa0850e..1478b0569 100644 --- a/js/src/wrappers/oai.ts +++ b/js/src/wrappers/oai.ts @@ -27,6 +27,7 @@ import { createLazyAPIPromise, EnhancedResponse, splitSpanInfo, + tracePromiseAsResponse, tracePromiseWithResponse, } from "./openai-promise-utils"; import { OpenAIV4Client } from "../vendor-sdk-types/openai-v4"; @@ -113,7 +114,6 @@ export function wrapOpenAIv4(openai: T): T { OpenAIModerationCreateParams, OpenAIModerationResponse >(typedOpenai.moderations, wrapModerations); - let betaProxy: OpenAIClient["beta"]; if (typedOpenai.beta?.chat?.completions?.stream) { const betaChatCompletionProxy = new Proxy( @@ -148,7 +148,6 @@ export function wrapOpenAIv4(openai: T): T { }); } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const topLevelProxy = new Proxy(typedOpenai, { get(target, name) { switch (name) { @@ -254,6 +253,12 @@ function wrapChatCompletion< // to avoid unhandled rejections when the underlying OpenAI call fails immediately. // Without lazy execution, the promise chain starts before error handlers are attached. let executionPromise: Promise> | null = null; + let apiPromise: APIPromise | null = null; + + const getAPIPromise = (): APIPromise => { + apiPromise ??= completion(params, options); + return apiPromise; + }; const ensureExecuted = (): Promise> => { if (!executionPromise) { @@ -265,37 +270,48 @@ function wrapChatCompletion< ); if (params.stream) { - const completionPromise = completion( - params, - options, - ) as APIPromise; - const { data, response } = await tracePromiseWithResponse( - openAIChannels.chatCompletionsCreate, - traceContext, - completionPromise, - ); + const completionPromise = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + getAPIPromise() as APIPromise; + const { data, response, request_id } = + await tracePromiseWithResponse( + openAIChannels.chatCompletionsCreate, + traceContext, + completionPromise, + ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { data: data as C, response }; + return { data: data as C, response, request_id }; } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const completionResponse = completion( - params, - options, - ) as APIPromise; - const { data, response } = await tracePromiseWithResponse( + const completionResponse = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + getAPIPromise() as APIPromise; + const { data, response, request_id } = await tracePromiseWithResponse( openAIChannels.chatCompletionsCreate, traceContext, completionResponse, ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { data: data as C, response }; + return { data: data as C, response, request_id }; })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + openAIChannels.chatCompletionsCreate, + createChannelContext( + openAIChannels.chatCompletionsCreate, + params, + span_info, + ), + getAPIPromise(), + ), + getAPIPromise, + ); }; } @@ -338,6 +354,11 @@ function wrapApiCreateWithChannel< >(allParams); let executionPromise: Promise>> | null = null; + let apiPromise: APIPromise> | null = null; + const getAPIPromise = () => { + apiPromise ??= create(params, options); + return apiPromise; + }; const ensureExecuted = () => { if (!executionPromise) { executionPromise = (async () => { @@ -345,13 +366,22 @@ function wrapApiCreateWithChannel< return tracePromiseWithResponse( channel, traceContext, - create(params, options), + getAPIPromise(), ); })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + channel, + createChannelContext(channel, params, span_info), + getAPIPromise(), + ), + getAPIPromise, + ); }; } diff --git a/js/src/wrappers/oai_responses.ts b/js/src/wrappers/oai_responses.ts index 8e936ebbc..466856286 100644 --- a/js/src/wrappers/oai_responses.ts +++ b/js/src/wrappers/oai_responses.ts @@ -11,6 +11,7 @@ import { createLazyAPIPromise, EnhancedResponse, splitSpanInfo, + tracePromiseAsResponse, tracePromiseWithResponse, } from "./openai-promise-utils"; @@ -78,6 +79,12 @@ function wrapResponsesAsync< let executionPromise: Promise>> | null = null; + let apiPromise: APIPromise> | null = null; + + const getAPIPromise = () => { + apiPromise ??= target(params, options); + return apiPromise; + }; const ensureExecuted = (): Promise< EnhancedResponse> @@ -85,15 +92,27 @@ function wrapResponsesAsync< if (!executionPromise) { executionPromise = (async () => { const traceContext = createChannelContext(channel, params, span_info); - const apiPromise = target(params, options); - return tracePromiseWithResponse(channel, traceContext, apiPromise); + return tracePromiseWithResponse( + channel, + traceContext, + getAPIPromise(), + ); })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + channel, + createChannelContext(channel, params, span_info), + getAPIPromise(), + ), + getAPIPromise, + ); }; } diff --git a/js/src/wrappers/openai-promise-utils.ts b/js/src/wrappers/openai-promise-utils.ts index afa23586c..9ba00d0bd 100644 --- a/js/src/wrappers/openai-promise-utils.ts +++ b/js/src/wrappers/openai-promise-utils.ts @@ -11,10 +11,12 @@ import type { export type EnhancedResponse = { response: Response; data: T; + request_id?: string | null; }; export interface APIPromise extends Promise { withResponse(): Promise>; + asResponse(): Promise; } type ChannelContext = @@ -71,19 +73,78 @@ export async function tracePromiseWithResponse< throw new Error("Expected withResponse() to provide response"); } - return { data, response: enhancedResponse.response }; + return { + data, + response: enhancedResponse.response, + request_id: enhancedResponse.request_id, + }; +} + +export async function tracePromiseAsResponse< + TChannel extends OpenAIAsyncChannel, + TResult extends ResultOf, +>( + channel: TChannel, + traceContext: ChannelContext, + apiPromise: APIPromise, +): Promise { + const tracePromise = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + channel.tracePromise as unknown as ( + fn: () => Promise, + context: ChannelContext, + ) => Promise; + + let response: Response | undefined; + await tracePromise(async () => { + response = await apiPromise.asResponse(); + traceContext.response = response; + return undefined; + }, traceContext); + + if (!response) { + throw new Error("Expected asResponse() to provide response"); + } + return response; } export function createLazyAPIPromise( ensureExecuted: () => Promise>, + ensureResponse: () => Promise, + getAPIPromise: () => APIPromise, ): APIPromise { - let dataPromise: Promise | null = null; + let firstConsumption: "data" | "response" | undefined; + let enhancedResponsePromise: Promise> | undefined; + let dataPromise: Promise | undefined; + let responsePromise: Promise | undefined; + + const withResponse = () => { + firstConsumption ??= "data"; + enhancedResponsePromise ??= + firstConsumption === "data" + ? ensureExecuted() + : getAPIPromise().withResponse(); + return enhancedResponsePromise; + }; + + const asResponse = () => { + firstConsumption ??= "response"; + responsePromise ??= + firstConsumption === "response" + ? ensureResponse() + : getAPIPromise().asResponse(); + return responsePromise; + }; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return new Proxy({} as APIPromise, { get(target, prop, receiver) { if (prop === "withResponse") { - return () => ensureExecuted(); + return withResponse; + } + + if (prop === "asResponse") { + return asResponse; } if ( @@ -92,9 +153,7 @@ export function createLazyAPIPromise( prop === "finally" || prop in Promise.prototype ) { - if (!dataPromise) { - dataPromise = ensureExecuted().then((result) => result.data); - } + dataPromise ??= withResponse().then((result) => result.data); const value = Reflect.get(dataPromise, prop, receiver); return typeof value === "function" ? value.bind(dataPromise) : value; } diff --git a/js/util/generated_plain_types.ts b/js/util/generated_plain_types.ts index b03f1fddc..dcaf1935a 100644 --- a/js/util/generated_plain_types.ts +++ b/js/util/generated_plain_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 776b6c295e852681) -- do not modify +// Auto-generated file (content hash e392acbca346c32a) -- do not modify export type AclObjectTypeType = /** @@ -773,6 +773,26 @@ export type FacetPreprocessorIdType = code: string; } | null; +export type SavedFunctionIdType = + | { + /** + * @enum function + */ + type: "function"; + id: string; + version?: /** + * The version of the function + */ + string | undefined; + } + | { + /** + * @enum global + */ + type: "global"; + name: string; + function_type: FunctionTypeEnumType; + }; export type TopicMapGenerationSettingsType = { /** * @enum hdbscan, kmeans, community @@ -795,9 +815,15 @@ export type TopicMapDataType = { */ type: "topic_map"; /** - * The facet field name to use as input for classification + * Materialized facet field name used when source_facet_function is absent */ source_facet: string; + source_facet_function?: + | (SavedFunctionIdType & + /** + * The stable function reference for the source facet + */ unknown) + | undefined; /** * The embedding model to use for embedding facet values */ @@ -1517,26 +1543,6 @@ export type ObjectReferenceNullishType = * Indicates the event was copied from another object. */ | null; -export type SavedFunctionIdType = - | { - /** - * @enum function - */ - type: "function"; - id: string; - version?: /** - * The version of the function - */ - string | undefined; - } - | { - /** - * @enum global - */ - type: "global"; - name: string; - function_type: FunctionTypeEnumType; - }; export type DatasetEventType = { /** * A unique identifier for the dataset event. If you don't provide one, Braintrust will generate one for you @@ -2748,7 +2754,7 @@ export type PromptParserNullishType = { } | null; export type PreprocessorSavedFunctionIdType = /** - * For prompt-backed scorers: the preprocessor function to use for trace template variables. Set to null to disable preprocessing. If omitted, the traced project's default preprocessor will be used, falling back to the global 'thread' preprocessor. + * For prompt-backed functions: the preprocessor function to use for trace template variables. Set to null to disable preprocessing. If omitted, the traced project's default preprocessor will be used, falling back to the global 'thread' preprocessor. */ | { /** @@ -4946,7 +4952,7 @@ export type TopicAutomationConfigType = { topic_map_functions: Array; scope?: | /** - * Execution scope for topic automation. Defaults to span-level execution. + * Execution scope for topic automation. */ (SpanScopeType | TraceScopeType | GroupScopeType | null) | undefined; @@ -6393,6 +6399,12 @@ export type RunEvalType = { */ | { data: Array; + } + /** + * Experiment whose inputs and outputs should be used as dataset inputs and expected values + */ + | { + experiment_name: string; }; name?: /** * The name of the eval to run when multiple evals available diff --git a/js/util/generated_types.ts b/js/util/generated_types.ts index fd807dcc5..ebf0f4be8 100644 --- a/js/util/generated_types.ts +++ b/js/util/generated_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 776b6c295e852681) -- do not modify +// Auto-generated file (content hash e392acbca346c32a) -- do not modify import { z } from "zod/v3"; @@ -273,6 +273,19 @@ export const FacetPreprocessorId = z.union([ z.null(), ]); export type FacetPreprocessorIdType = z.infer; +export const SavedFunctionId = z.union([ + z.object({ + type: z.literal("function"), + id: z.string(), + version: z.string().optional(), + }), + z.object({ + type: z.literal("global"), + name: z.string(), + function_type: FunctionTypeEnum.optional().default("scorer"), + }), +]); +export type SavedFunctionIdType = z.infer; export const TopicMapGenerationSettings = z.object({ algorithm: z.enum(["hdbscan", "kmeans", "community"]), dimension_reduction: z.enum(["umap", "pca", "none"]), @@ -289,6 +302,7 @@ export type TopicMapGenerationSettingsType = z.infer< export const TopicMapData = z.object({ type: z.literal("topic_map"), source_facet: z.string(), + source_facet_function: SavedFunctionId.and(z.unknown()).optional(), embedding_model: z.string(), bundle_key: z.string().optional(), report_key: z.string().optional(), @@ -629,19 +643,6 @@ export const ObjectReferenceNullish = z.union([ z.null(), ]); export type ObjectReferenceNullishType = z.infer; -export const SavedFunctionId = z.union([ - z.object({ - type: z.literal("function"), - id: z.string(), - version: z.string().optional(), - }), - z.object({ - type: z.literal("global"), - name: z.string(), - function_type: FunctionTypeEnum.optional().default("scorer"), - }), -]); -export type SavedFunctionIdType = z.infer; export const DatasetEvent = z.object({ id: z.string(), _xact_id: z.string(), @@ -2170,6 +2171,7 @@ export const RunEval = z.object({ .optional(), }), z.object({ data: z.array(z.unknown()) }), + z.object({ experiment_name: z.string() }), ]), name: z.string().optional(), parameters: z.object({}).partial().passthrough().optional(),